{"cells":[{"cell_type":"markdown","source":["# Text2Phoneme Transformer \n","In this notebook we build a model that converts sentences to phoneme sequences using a transformer model. The model is trained on the timit_asr dataset and several experiments are run to evaluate it's performance"],"metadata":{"id":"alETo-QH5lwH"}},{"cell_type":"markdown","source":["## Setup\n","1. connect the notebook to drive to load custom class\n","2. load necessary pip packages\n","3. load the timit_asr dataset"],"metadata":{"id":"Dv7kT8J-5y-V"}},{"cell_type":"code","execution_count":1,"metadata":{"id":"ujH-VHtsOOoy","executionInfo":{"status":"ok","timestamp":1639669459354,"user_tz":300,"elapsed":6,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}}},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":12965,"status":"ok","timestamp":1639669472314,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"},"user_tz":300},"id":"WKLqYVqQOXD_","outputId":"725369be-ae08-476a-f59d-e20ae234ad47"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":3,"metadata":{"id":"h5ImY5dOOaou","executionInfo":{"status":"ok","timestamp":1639669472315,"user_tz":300,"elapsed":6,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}}},"outputs":[],"source":["import os\n","import sys\n","\n","WORKSPACE_DIR = 'college/595_final_project'\n","WORKSPACE_PATH = os.path.join('drive', 'My Drive', WORKSPACE_DIR)\n","sys.path.append(WORKSPACE_PATH)"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":10546,"status":"ok","timestamp":1639669482857,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"},"user_tz":300},"id":"NYjgqG0QPJp2","outputId":"f38ccd7c-a708-4d4f-9330-183ddd930b5c"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting datasets\n","  Downloading datasets-1.16.1-py3-none-any.whl (298 kB)\n","\u001b[K     |████████████████████████████████| 298 kB 8.0 MB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n","Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n","Collecting huggingface-hub<1.0.0,>=0.1.0\n","  Downloading huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n","\u001b[K     |████████████████████████████████| 61 kB 662 kB/s \n","\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n","Collecting xxhash\n","  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n","\u001b[K     |████████████████████████████████| 243 kB 54.1 MB/s \n","\u001b[?25hCollecting fsspec[http]>=2021.05.0\n","  Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n","\u001b[K     |████████████████████████████████| 132 kB 59.8 MB/s \n","\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n","Collecting aiohttp\n","  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n","\u001b[K     |████████████████████████████████| 1.1 MB 53.5 MB/s \n","\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n","Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n","Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.13)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n","Collecting async-timeout<5.0,>=4.0.0a3\n","  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n","Collecting frozenlist>=1.1.1\n","  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n","\u001b[K     |████████████████████████████████| 192 kB 62.9 MB/s \n","\u001b[?25hCollecting multidict<7.0,>=4.5\n","  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n","\u001b[K     |████████████████████████████████| 160 kB 65.2 MB/s \n","\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n","Collecting aiosignal>=1.1.2\n","  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n","Collecting yarl<2.0,>=1.0\n","  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n","\u001b[K     |████████████████████████████████| 271 kB 75.4 MB/s \n","\u001b[?25hCollecting asynctest==0.13.0\n","  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n","Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, huggingface-hub, datasets\n","Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 datasets-1.16.1 frozenlist-1.2.0 fsspec-2021.11.1 huggingface-hub-0.2.1 multidict-5.2.0 xxhash-2.0.2 yarl-1.7.2\n","Collecting pytorch-nlp\n","  Downloading pytorch_nlp-0.5.0-py3-none-any.whl (90 kB)\n","\u001b[K     |████████████████████████████████| 90 kB 5.5 MB/s \n","\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from pytorch-nlp) (1.19.5)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from pytorch-nlp) (4.62.3)\n","Installing collected packages: pytorch-nlp\n","Successfully installed pytorch-nlp-0.5.0\n"]}],"source":["!pip install datasets\n","!pip install pytorch-nlp"]},{"cell_type":"code","execution_count":9,"metadata":{"id":"0zV6_LzLOe1N","colab":{"base_uri":"https://localhost:8080/","height":136,"referenced_widgets":["bc5d01f9505d4c5692b116bf5cbfd921","c28803a8b7054413860d06836194034c","41b8c2ac6da44b7399b1076305509855","61865673211f4c6ea73ff6f346e171a1","b34d81cfe056497997680a584df615ac","4b283511714842559d41fb8f31f01bb2","02904e539c3a494393986af5cee371b9","73482b9190a54436890d91e304185d81","6f23ddc39ea2440ca00a3c646aee962d","272024c6238a4b1cbe888d547627898f","0f5005881e4c46f48c963ba266144fbc","d06641948886478d8bf7185c22f2591b","14894674f58c417bb3c3a5d12205cffe","edeb40b5644741fe9b9249cd1321907d","6e4b968aeb5a4a849d37889eb04fb3ba","0b82c21e2f9843b7bc7ec7396d901a79","c0b7d520a7ae476890a104e36bb7fef0","a47bec8da73943208bc2b4dda0abc0d5","7bcf2aae88e342dcb0a81d122f831521","c9fb8e4168bd4a8ebc1b2e4e685034d5","3eb9ddcd25774487b656cdb048f25720","392598e80294491ba1a1eed8de5b3580","3c1a14e1b4734d7db2b9a1471610d50f","9cf4d0271d2f4eeb8343a07d12f025b6","31a3b4ccbd554fca8ca880dde2ef37cb","b9f23d9094ba42458f84e7d35d178d60","fef496f780f44745ad6a4bfd5b6718b7","ed578ff7c89e4839a4b3548764309757","81bfe3fef60942e4bdacfda857bad23b","506b9c939a60448881c585bd54a3af14","8dacfff7491c4b11a9c31940b8d66310","fd48d8b2ea884309bf02e8fdf40922a7","c8d5396762d347b6b3f0f394263ca249","76b19283c866471a96fdfb74104f8c95","79c5f718ebc048b9bd14e50e15556d6a","f220dbc1f7cf460494fa380ea9187c03","ef8e8158fcc24f8797b126543d51765f","af57205ab6c54e9eac09dd515eff85f2","78dd47cd43864c228f36d05ebe845a3c","85d7817bcf554456868e1a6e762e627f","fc48e1f1b2534f94838c4f56c98fa806","426e2d7bb3bd4bd4beb8cbed3ec22320","3fbd2ec4ae8b4373bca3ebac00230727","09ee26670cff4b908ea97fbec065c442"]},"executionInfo":{"status":"ok","timestamp":1639669681770,"user_tz":300,"elapsed":73659,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"301bf147-0b0d-4e7b-b1d4-03228cb6ce7b"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading and preparing dataset timit_asr/clean (download: 828.75 MiB, generated: 7.90 MiB, post-processed: Unknown size, total: 836.65 MiB) to /root/.cache/huggingface/datasets/timit_asr/clean/2.0.1/5bebea6cd9df0fc2c8c871250de23293a94c1dc49324182b330b6759ae6718f8...\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"bc5d01f9505d4c5692b116bf5cbfd921","version_minor":0,"version_major":2},"text/plain":["Downloading:   0%|          | 0.00/869M [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d06641948886478d8bf7185c22f2591b","version_minor":0,"version_major":2},"text/plain":["0 examples [00:00, ? examples/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3c1a14e1b4734d7db2b9a1471610d50f","version_minor":0,"version_major":2},"text/plain":["0 examples [00:00, ? examples/s]"]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Dataset timit_asr downloaded and prepared to /root/.cache/huggingface/datasets/timit_asr/clean/2.0.1/5bebea6cd9df0fc2c8c871250de23293a94c1dc49324182b330b6759ae6718f8. Subsequent calls will reuse this data.\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"76b19283c866471a96fdfb74104f8c95","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/2 [00:00<?, ?it/s]"]},"metadata":{}}],"source":["import torch\n","import torch.nn as nn\n","from torch import optim\n","from tqdm import tqdm\n","import torch.nn.functional as F\n","import time\n","import math\n","import matplotlib.pyplot as plt\n","import matplotlib.ticker as ticker\n","import numpy as np\n","import random\n","from datasets import load_dataset\n","\n","DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","timit_dataset = load_dataset('timit_asr')"]},{"cell_type":"code","source":["from torchnlp.word_to_vector import GloVe\n","vocab_to_glove = GloVe(name='6B', dim=300)"],"metadata":{"id":"PGHqOA1RRvgO"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Data Processing Functions\n","In this section we define the following functions to process the timit_asr dataset:\n","\n","`map_phonemes_to_words` - defines a mapping between each phoneme in the  phoneme sequence to the word it came from in the original sentence\n","\n","`preprocess_timit_data` - process the huggingface dataset object and store each datapoint as a triplet of sentence, phoneme sequence, and phoneme word mappings\n","\n","`vectorize_seq` - converts a sequence of tokens to a pytorch tensor\n","\n","`vectorize_mappings` - converts a list of phoneme to word mappings to a pytorch tensor \n","\n","`vectorize_pair` - returns a vectorized representation of a preprocessed timit_asr datapoint\n","\n","`build_dataset` - creates a pytorch dataset object from the preprocessed timit_asr data"],"metadata":{"id":"tGPhnois590w"}},{"cell_type":"code","source":["def map_phonemes_to_words(dp):\n","  word_idx = 0\n","  phoneme_idx = 0\n","  mappings = []\n","  for word_stop_time in dp['word_detail']['stop']:\n","    while word_stop_time >= dp['phonetic_detail']['stop'][phoneme_idx]:\n","      mappings.append(word_idx)\n","      phoneme_idx += 1\n","    word_idx += 1\n","\n","  while phoneme_idx < len(dp['phonetic_detail']['utterance']):\n","    mappings.append(word_idx-1)\n","    phoneme_idx += 1\n","\n","  phonetic_len = len(dp['phonetic_detail']['utterance'])\n","  assert len(mappings) == len(dp['phonetic_detail']['utterance']), f'ASSERT FAILED: mapping len: {len(mappings)} vs phoneme len: {phonetic_len}'\n","  return mappings\n","\n","def preprocess_timit_data(data):\n","  pairs = []\n","  for dp in data:\n","    word_seq = dp['word_detail']['utterance']\n","    phoneme_seq = dp['phonetic_detail']['utterance']\n","    phoneme_word_mappings = map_phonemes_to_words(dp)\n","    pairs.append((word_seq, phoneme_seq, phoneme_word_mappings))\n","  return pairs"],"metadata":{"id":"cBKV1hKG6Yj2","executionInfo":{"status":"ok","timestamp":1639669698020,"user_tz":300,"elapsed":146,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["print(timit_dataset['train'][4497]['word_detail']['utterance'])\n","print(timit_dataset['train'][4497]['phonetic_detail']['utterance'][1:-1])\n","print(map_phonemes_to_words(timit_dataset['train'][4497])[1:-1])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kCXb8iw065no","executionInfo":{"status":"ok","timestamp":1639671215943,"user_tz":300,"elapsed":160,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"df01d91d-560a-4dff-ae69-4f317af08600"},"execution_count":19,"outputs":[{"output_type":"stream","name":"stdout","text":["['such', 'legislation', 'was', 'clarified', 'and', 'extended', 'from', 'time', 'to', 'time', 'thereafter']\n","['s', 'ah', 'sh', 'l', 'eh', 'dcl', 'jh', 'ax', 's', 'l', 'ey', 'sh', 'en', 'w', 'ax', 'z', 'kcl', 'k', 'l', 'eh', 'axr', 'f', 'ay', 'dcl', 'en', 'ix', 'kcl', 'k', 's', 'tcl', 't', 'eh', 'n', 'd', 'ix', 'dcl', 'f', 'em', 'tcl', 't', 'ay', 'm', 'tcl', 't', 'ax', 'tcl', 't', 'ay', 'm', 'dh', 'eh', 'r', 'ae', 'f', 'tcl', 't', 'axr']\n","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10]\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"id":"yv4ObuxFbPn5"},"outputs":[],"source":["def vectorize_seq(token_encoder, sequence):\n","  indexes = [token_encoder.bos_token_id] + [token_encoder.get_token_index(token) for token in sequence] + [token_encoder.eos_token_id]\n","  if len(indexes) < MAX_LEN:\n","    indexes = indexes + [token_encoder.pad_token_id] * (MAX_LEN - len(indexes))\n","  else:\n","    indexes = indexes[:MAX_LEN]\n","  return torch.tensor(indexes, dtype=torch.long, device=DEVICE)\n","\n","def glove_vectorize_seq(token_encoder, sequence):\n","  embedding_matrix = torch.zeros((MAX_LEN, HID_DIM), dtype=torch.float32, device=DEVICE)\n","  sum = 0\n","  for word_idx, word in enumerate(sequence):\n","    if word_idx == MAX_LEN:\n","      break\n","    if token_encoder.get_token_index(word) == token_encoder.unk_token_id:\n","      embedding_matrix[word_idx] = 1\n","    word = word.replace(\"'\", \"\")\n","    if word not in vocab_to_glove:\n","      embedding_matrix[word_idx] = 1\n","    else:\n","      embedding_matrix[word_idx] = vocab_to_glove[word][:HID_DIM]\n","  return embedding_matrix\n","\n","\n","def vectorize_mappings(indexes):\n","  indexes = [0] + [i + 1 for i in indexes] + [indexes[-1]+1]\n","  if len(indexes) < MAX_LEN:\n","    indexes = indexes + [MAX_LEN+1] * (MAX_LEN - len(indexes))\n","  else:\n","    indexes = indexes[:MAX_LEN]\n","  return torch.tensor(indexes, dtype=torch.long, device=DEVICE)\n","\n","\n","def vectorize_pair(pair, use_glove=False):\n","  if use_glove:\n","    word_vector = glove_vectorize_seq(word_encoder, pair[0])\n","  else:\n","    word_vector = vectorize_seq(word_encoder, pair[0])\n","\n","  phoneme_vector = vectorize_seq(phoneme_encoder, pair[1])\n","  phone_word_mapping_vector = vectorize_mappings(pair[2])\n","  return word_vector, phoneme_vector, phone_word_mapping_vector"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"9CknOIsXdC2T"},"outputs":[],"source":["from torch.utils.data import TensorDataset, DataLoader\n","\n","def build_dataset(pairs):\n","  word_vecs = torch.ones((len(pairs), MAX_LEN), dtype=torch.long)\n","  phoneme_vecs = torch.ones((len(pairs), MAX_LEN), dtype=torch.long)\n","  phoneme_word_mapping_vecs = torch.ones((len(pairs), MAX_LEN), dtype=torch.long)\n","  for idx, pair in enumerate(pairs):\n","    word_vec, phoneme_vec, mapping_vec = vectorize_pair(pair)\n","    word_vecs[idx] = word_vec\n","    phoneme_vecs[idx] = phoneme_vec\n","    phoneme_word_mapping_vecs[idx] = mapping_vec\n","\n","  return TensorDataset(word_vecs, phoneme_vecs, phoneme_word_mapping_vecs)\n","\n","def build_glove_dataset(pairs):\n","  word_vecs = torch.ones((len(pairs), MAX_LEN, HID_DIM), dtype=torch.long)\n","  phoneme_vecs = torch.ones((len(pairs), MAX_LEN), dtype=torch.long)\n","  phoneme_word_mapping_vecs = torch.ones((len(pairs), MAX_LEN), dtype=torch.long)\n","  for idx, pair in enumerate(pairs):\n","    word_vec, phoneme_vec, mapping_vec = vectorize_pair(pair, use_glove=True)\n","    word_vecs[idx] = word_vec\n","    phoneme_vecs[idx] = phoneme_vec\n","    phoneme_word_mapping_vecs[idx] = mapping_vec\n","\n","  return TensorDataset(word_vecs, phoneme_vecs, phoneme_word_mapping_vecs)"]},{"cell_type":"markdown","source":["## Training Functions\n","Here we defined the functions we need for training the model. The model itself is defined in `transformers.py`\n","\n","`initialize_weights` - initializes the weights for a seq2seq transformer model\n","\n","`train` - contains the core training code\n","\n","`evaluate` - evaluates the model\n","\n","`get_metrics` - provides model performance metrics on a dataset\n","\n","`get_samples` - generates samples given a model"],"metadata":{"id":"ySPgc0-17d8Y"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"uokkDO-1hmMj"},"outputs":[],"source":["def initialize_weights(m):\n","  if hasattr(m, 'weight') and m.weight.dim() > 1:\n","    nn.init.xavier_uniform_(m.weight.data)\n","\n","def train(model, iterator, optimizer, vocab_criterion, mapping_criterion, clip):\n","  model.train()\n","  \n","  epoch_loss = 0\n","  \n","  for i, batch in enumerate(iterator):\n","      \n","    src = batch[0].to(DEVICE) \n","    trg = batch[1].to(DEVICE) \n","    trg_mapping = batch[2].to(DEVICE)\n","    \n","    optimizer.zero_grad()\n","    \n","    output, output_mapping, _ = model(src, trg[:,:-1])\n","    output_dim = output.shape[-1]\n","    output_mapping_dim = output_mapping.shape[-1]\n","        \n","    output = output.contiguous().view(-1, output_dim)\n","    trg = trg[:,1:].contiguous().view(-1)\n","\n","    output_mapping = output_mapping.contiguous().view(-1, output_mapping_dim)\n","    trg_mapping = trg_mapping[:,1:].contiguous().view(-1)\n","\n","\n","    loss = vocab_criterion(output, trg) # + mapping_criterion(output_mapping, trg_mapping) \n","    loss.backward()\n","    torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n","    \n","    optimizer.step()\n","    \n","    epoch_loss += loss.item()\n","        \n","  return epoch_loss / len(iterator)"]},{"cell_type":"code","source":["from nltk.translate import bleu\n","from nltk.translate.bleu_score import SmoothingFunction\n","\n","def epoch_time(start_time, end_time):\n","  elapsed_time = end_time - start_time\n","  elapsed_mins = int(elapsed_time / 60)\n","  elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n","  return elapsed_mins, elapsed_secs\n","\n","\n","def edit_score(pred, truth, padding_token = None):\n","    # if the lists are padded, then remove the padding before calculating edit distance\n","    if padding_token:\n","        if padding_token in pred:\n","            pred = pred[:pred.index(padding_token)]\n","\n","        if padding_token in truth:\n","            truth = truth[:pred.index(padding_token)]\n","\n","        print(pred, truth)\n","\n","    m = len(pred)\n","    n = len(truth)\n","    # Create a table to store results of subproblems\n","    dp = [[0 for x in range(n + 1)] for x in range(m + 1)]\n","  \n","    # Fill d[][] in bottom up manner\n","    for i in range(m + 1):\n","        for j in range(n + 1):\n","  \n","            # If first list is empty, only option is to\n","            # insert all elements of second list\n","            if i == 0:\n","                dp[i][j] = j    # Min. operations = j\n","  \n","            # If second list is empty, only option is to\n","            # remove all elements of second list\n","            elif j == 0:\n","                dp[i][j] = i    # Min. operations = i\n","  \n","            # If last elements are same, ignore last char\n","            # and recur for remaining list\n","\n","            elif pred[i - 1] == truth[j - 1]:\n","                dp[i][j] = dp[i - 1][j - 1]\n","  \n","            # If last elements are different, consider all\n","            # possibilities and find minimum\n","            else:\n","                dp[i][j] = 1 + min(dp[i][j - 1],        # Insert\n","                                   dp[i - 1][j],        # Remove\n","                                   dp[i - 1][j - 1])    # Replace\n","  \n","    # normalize the edit distance by dividing the number edit distance by the the length of truth\n","    return dp[m][n] / n\n","\n","def accuracy_fn(ref, pred, index=None):\n","  if index:\n","    ref = ref[: min(len(ref), index)]\n","    pred = pred[: min(len(pred), index)]\n","  if len(ref) > len(pred):\n","    pred = pred + [-1] * (len(ref) - len(pred))\n","  if len(pred) > len(ref):\n","    pred = pred[: len(pred)]    \n","\n","  return sum(1 for x,y in zip(ref, pred) if x == y) / len(ref)\n","\n","def evaluate(model, dataloader, vocab_criterion, mapping_criterion):\n","\n","  smoothie = SmoothingFunction().method4\n","  \n","  bleu_score = 0\n","  phoneme_accuracy = 0\n","  edit_dist = 0\n","  vocab_loss = 0\n","  mapping_loss = 0\n","  num_datapoints = 0\n","\n","  mapping_ed = 0\n","  mapping_acc = 0\n","\n","  model.eval()\n","  with torch.no_grad():\n","    for i, batch in enumerate(dataloader):\n","      src = batch[0].to(DEVICE) # (seq_len, batch_size)\n","      trg = batch[1].to(DEVICE) # (seq_len, batch_size)\n","      trg_mapping = batch[2].to(DEVICE)\n","\n","      optimizer.zero_grad()\n","          \n","      output, output_mapping, _ = model(src, trg[:,:-1]) # (batch size, trg len - 1, output dim)\n","      output_dim = output.shape[-1]\n","      output_mapping_dim = output_mapping.shape[-1]\n","\n","      vocab_loss += vocab_criterion(output.reshape(-1, output_dim), trg[:,1:].reshape(-1))\n","      mapping_loss += mapping_criterion(output_mapping.reshape(-1, output_mapping_dim), trg_mapping[:,1:].reshape(-1))\n","\n","\n","      output = output.argmax(-1)\n","      output_mapping = output_mapping.argmax(-1)\n","\n","      for j in range(len(batch[0])):\n","        num_datapoints += 1\n","\n","        predicted_sequence = phoneme_encoder.decode_sequence(output[j].tolist())\n","        target_sequence = phoneme_encoder.decode_sequence(trg[j].tolist())[1:]\n","\n","        predicted_mapping = output_mapping[j].tolist()\n","        target_mapping = trg_mapping[j].tolist()[1:]\n","\n","        if phoneme_encoder.eos_token in predicted_sequence:\n","          predicted_sequence = predicted_sequence[:predicted_sequence.index(phoneme_encoder.eos_token)]\n","        target_len = len(target_sequence)\n","        if phoneme_encoder.eos_token in target_sequence:\n","          target_len = target_sequence.index(phoneme_encoder.eos_token)\n","          target_sequence = target_sequence[:target_len]\n","\n","        predicted_mapping = predicted_mapping[:target_len]   \n","        target_mapping = target_mapping[:target_len]   \n","\n","        bleu_score += bleu([target_sequence], predicted_sequence, smoothing_function=smoothie)\n","        phoneme_accuracy += accuracy_fn(target_sequence, predicted_sequence)\n","        edit_dist += edit_score(target_sequence, predicted_sequence)\n","\n","        mapping_acc += accuracy_fn(target_mapping, predicted_mapping)\n","        mapping_ed += edit_score(target_mapping, predicted_mapping)        \n","\n","  return {\n","      'phoneme_acc': phoneme_accuracy/num_datapoints, \n","      'phoneme_bleu': bleu_score/num_datapoints,\n","      'phoneme_edit': edit_dist/num_datapoints,\n","      'mapping_acc': mapping_acc/num_datapoints,\n","      'mapping_ed': mapping_ed/num_datapoints,\n","      'vocab_loss': vocab_loss.cpu().numpy() / len(dataloader)\n","  }\n","\n","def generate_samples(model, dataloader):\n","  smoothie = SmoothingFunction().method4\n","  samples = []\n","\n","  with torch.no_grad():\n","    for i, batch in enumerate(dataloader):\n","      src = batch[0].to(DEVICE) # (seq_len, batch_size)\n","      trg = batch[1].to(DEVICE) # (seq_len, batch_size)\n","      trg_mapping = batch[2].to(DEVICE)\n","\n","\n","      optimizer.zero_grad()\n","          \n","      output, output_mapping, _ = model(src, trg[:,:-1]) # (batch size, trg len - 1, output dim)\n","      output = output.argmax(-1)\n","      output_mapping = output_mapping.argmax(-1)\n","\n","                \n","      for j in range(len(batch[0])):\n","\n","        predicted_sequence = phoneme_encoder.decode_sequence(output[j].tolist())\n","        target_sequence = phoneme_encoder.decode_sequence(trg[j].tolist())[1:]\n","\n","\n","        predicted_mapping = output_mapping[j].tolist()\n","        target_mapping = trg_mapping[j].tolist()[1:]\n","\n","        if phoneme_encoder.eos_token in predicted_sequence:\n","          predicted_sequence = predicted_sequence[:predicted_sequence.index(phoneme_encoder.eos_token)]\n","\n","        target_len = len(target_sequence)\n","        if phoneme_encoder.eos_token in target_sequence:\n","          target_len = target_sequence.index(phoneme_encoder.eos_token)\n","          target_sequence = target_sequence[:target_len]\n","\n","        predicted_mapping = predicted_mapping[:target_len]   \n","        target_mapping = target_mapping[:target_len]            \n","\n","        phoneme_accuracy = accuracy_fn(target_sequence, predicted_sequence)\n","        bleu_score = bleu([target_sequence], predicted_sequence, smoothing_function=smoothie)\n","        edit_dist = edit_score(target_sequence, predicted_sequence)\n","\n","        mapping_acc = accuracy_fn(target_mapping, predicted_mapping)\n","        mapping_ed = edit_score(target_mapping, predicted_mapping)             \n","\n","        samples.append({\n","            'target': target_sequence,\n","            'predicted': predicted_sequence,\n","            'target_mapping': target_mapping,\n","            'predicted_mapping': predicted_mapping,\n","            'phoneme_accuracy': phoneme_accuracy,\n","            'phoneme_bleu': bleu_score,\n","            'phoneme_edit': edit_dist,\n","            'mapping_ed': mapping_ed,\n","            'mapping_acc': mapping_acc,\n","        })\n","\n","  return samples"],"metadata":{"id":"_Xy9jPehZWqp"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Training Paramaters"],"metadata":{"id":"jdtElCdf802h"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"qRQpQqfhZBrO"},"outputs":[],"source":["# GLOBALS\n","MAX_LEN = 72\n","\n","# MODEL HYPERPARAMS\n","HID_DIM = 256\n","ENC_LAYERS = 3\n","DEC_LAYERS = 3\n","ENC_HEADS = 8\n","DEC_HEADS = 8\n","ENC_PF_DIM = 512\n","DEC_PF_DIM = 512\n","\n","# TRAINING PARAMS\n","BATCH_SIZE = 128\n","N_EPOCHS = 50\n","LEARNING_RATE = 0.0005\n","\n","\n","# REGULARIZATION PARAMATERS\n","ENC_DROPOUT = 0.1\n","DEC_DROPOUT = 0.1\n","CLIP = 1"]},{"cell_type":"markdown","source":["## Data Processing\n","\n","In this section we use the helper functions defined above to process the timit_asr dataset into a pytorch dataloader that we can feed into the seq2seq transformer model"],"metadata":{"id":"Dh0Kq39V8_gn"}},{"cell_type":"code","source":["from token_encoder import TokenEncoder, build_io_token_encodings\n","\n","train_pairs = preprocess_timit_data(timit_dataset['train'])\n","test_pairs = preprocess_timit_data(timit_dataset['test'])\n","\n","\n","word_encoder = TokenEncoder(MAX_LEN)\n","phoneme_encoder = TokenEncoder(MAX_LEN)\n","build_io_token_encodings(word_encoder, phoneme_encoder, train_pairs)\n","print(f'training points: {len(train_pairs)}, num input tokens: {word_encoder.n_tokens}, num output tokens: {phoneme_encoder.n_tokens}')\n","\n","# train_dataset = build_dataset(train_pairs)\n","train_dataset = build_glove_dataset(train_pairs)\n","train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE)\n","\n","# eval_dataset = build_dataset(test_pairs)\n","eval_dataset = build_glove_dataset(test_pairs)\n","eval_loader = DataLoader(eval_dataset, batch_size=BATCH_SIZE) "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MbXqvOoH4kNh","executionInfo":{"status":"ok","timestamp":1639666248072,"user_tz":300,"elapsed":4086,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"8878307b-366a-4763-ebba-63966de20e66"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["training points: 4620, num input tokens: 4897, num output tokens: 65\n"]}]},{"cell_type":"markdown","source":["## Model Training\n","Load and train a seq2seq transformer model"],"metadata":{"id":"FmbFfNnQ9M9A"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"CNiDbLoQj4HG"},"outputs":[],"source":["from transformer import Encoder, Decoder, Seq2Seq\n","\n","\n","enc = Encoder(word_encoder.n_tokens, \n","              HID_DIM, \n","              ENC_LAYERS, \n","              ENC_HEADS, \n","              ENC_PF_DIM, \n","              ENC_DROPOUT, \n","              DEVICE)\n","\n","dec = Decoder(phoneme_encoder.n_tokens, \n","              HID_DIM, \n","              DEC_LAYERS, \n","              DEC_HEADS, \n","              DEC_PF_DIM, \n","              DEC_DROPOUT, \n","              DEVICE)\n","model = Seq2Seq(enc, dec, word_encoder.pad_token_id, phoneme_encoder.pad_token_id, DEVICE).to(DEVICE)\n","model.apply(initialize_weights);\n","\n","\n","optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n","vocab_criterion = nn.CrossEntropyLoss(ignore_index = phoneme_encoder.pad_token_id)\n","mapping_criterion = nn.CrossEntropyLoss(ignore_index = MAX_LEN + 1)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TzXzApfyij30","executionInfo":{"status":"ok","timestamp":1639666720813,"user_tz":300,"elapsed":470032,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"cf43adc9-31c3-4397-ca2b-7ef53185a66c"},"outputs":[{"output_type":"stream","name":"stderr","text":["  2%|▏         | 1/50 [00:09<07:42,  9.44s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 01 | Time: 0m 9s\n","\tTrain Loss: 3.477 | Train PPL:  32.377\n","\t Val. Loss: 2.674 |  Val. PPL:  14.497\n"]},{"output_type":"stream","name":"stderr","text":["\r  4%|▍         | 2/50 [00:18<07:31,  9.41s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 02 | Time: 0m 9s\n","\tTrain Loss: 2.550 | Train PPL:  12.804\n","\t Val. Loss: 2.432 |  Val. PPL:  11.384\n"]},{"output_type":"stream","name":"stderr","text":["\r  6%|▌         | 3/50 [00:28<07:21,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 03 | Time: 0m 9s\n","\tTrain Loss: 2.373 | Train PPL:  10.735\n","\t Val. Loss: 2.361 |  Val. PPL:  10.607\n"]},{"output_type":"stream","name":"stderr","text":["\r  8%|▊         | 4/50 [00:37<07:12,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 04 | Time: 0m 9s\n","\tTrain Loss: 2.300 | Train PPL:   9.973\n","\t Val. Loss: 2.331 |  Val. PPL:  10.289\n"]},{"output_type":"stream","name":"stderr","text":["\r 10%|█         | 5/50 [00:47<07:03,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 05 | Time: 0m 9s\n","\tTrain Loss: 2.253 | Train PPL:   9.515\n","\t Val. Loss: 2.322 |  Val. PPL:  10.195\n"]},{"output_type":"stream","name":"stderr","text":["\r 12%|█▏        | 6/50 [00:56<06:53,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 06 | Time: 0m 9s\n","\tTrain Loss: 2.213 | Train PPL:   9.141\n","\t Val. Loss: 2.310 |  Val. PPL:  10.074\n"]},{"output_type":"stream","name":"stderr","text":["\r 14%|█▍        | 7/50 [01:05<06:45,  9.42s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 07 | Time: 0m 9s\n","\tTrain Loss: 2.175 | Train PPL:   8.806\n","\t Val. Loss: 2.331 |  Val. PPL:  10.289\n"]},{"output_type":"stream","name":"stderr","text":["\r 16%|█▌        | 8/50 [01:15<06:36,  9.44s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 08 | Time: 0m 9s\n","\tTrain Loss: 2.134 | Train PPL:   8.450\n","\t Val. Loss: 2.350 |  Val. PPL:  10.481\n"]},{"output_type":"stream","name":"stderr","text":["\r 18%|█▊        | 9/50 [01:24<06:26,  9.44s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 09 | Time: 0m 9s\n","\tTrain Loss: 2.089 | Train PPL:   8.078\n","\t Val. Loss: 2.365 |  Val. PPL:  10.645\n"]},{"output_type":"stream","name":"stderr","text":["\r 20%|██        | 10/50 [01:34<06:17,  9.44s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 10 | Time: 0m 9s\n","\tTrain Loss: 2.047 | Train PPL:   7.742\n","\t Val. Loss: 2.358 |  Val. PPL:  10.566\n"]},{"output_type":"stream","name":"stderr","text":["\r 22%|██▏       | 11/50 [01:43<06:08,  9.44s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 11 | Time: 0m 9s\n","\tTrain Loss: 1.999 | Train PPL:   7.382\n","\t Val. Loss: 2.405 |  Val. PPL:  11.081\n"]},{"output_type":"stream","name":"stderr","text":["\r 24%|██▍       | 12/50 [01:53<05:58,  9.43s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 12 | Time: 0m 9s\n","\tTrain Loss: 1.953 | Train PPL:   7.049\n","\t Val. Loss: 2.374 |  Val. PPL:  10.735\n"]},{"output_type":"stream","name":"stderr","text":["\r 26%|██▌       | 13/50 [02:02<05:48,  9.41s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 13 | Time: 0m 9s\n","\tTrain Loss: 1.904 | Train PPL:   6.710\n","\t Val. Loss: 2.424 |  Val. PPL:  11.289\n"]},{"output_type":"stream","name":"stderr","text":["\r 28%|██▊       | 14/50 [02:11<05:38,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 14 | Time: 0m 9s\n","\tTrain Loss: 1.860 | Train PPL:   6.426\n","\t Val. Loss: 2.412 |  Val. PPL:  11.151\n"]},{"output_type":"stream","name":"stderr","text":["\r 30%|███       | 15/50 [02:21<05:28,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 15 | Time: 0m 9s\n","\tTrain Loss: 1.819 | Train PPL:   6.164\n","\t Val. Loss: 2.418 |  Val. PPL:  11.221\n"]},{"output_type":"stream","name":"stderr","text":["\r 32%|███▏      | 16/50 [02:30<05:18,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 16 | Time: 0m 9s\n","\tTrain Loss: 1.773 | Train PPL:   5.886\n","\t Val. Loss: 2.415 |  Val. PPL:  11.184\n"]},{"output_type":"stream","name":"stderr","text":["\r 34%|███▍      | 17/50 [02:39<05:09,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 17 | Time: 0m 9s\n","\tTrain Loss: 1.726 | Train PPL:   5.619\n","\t Val. Loss: 2.448 |  Val. PPL:  11.562\n"]},{"output_type":"stream","name":"stderr","text":["\r 36%|███▌      | 18/50 [02:49<05:00,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 18 | Time: 0m 9s\n","\tTrain Loss: 1.690 | Train PPL:   5.417\n","\t Val. Loss: 2.476 |  Val. PPL:  11.895\n"]},{"output_type":"stream","name":"stderr","text":["\r 38%|███▊      | 19/50 [02:58<04:51,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 19 | Time: 0m 9s\n","\tTrain Loss: 1.648 | Train PPL:   5.196\n","\t Val. Loss: 2.489 |  Val. PPL:  12.050\n"]},{"output_type":"stream","name":"stderr","text":["\r 40%|████      | 20/50 [03:08<04:41,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 20 | Time: 0m 9s\n","\tTrain Loss: 1.609 | Train PPL:   4.998\n","\t Val. Loss: 2.516 |  Val. PPL:  12.378\n"]},{"output_type":"stream","name":"stderr","text":["\r 42%|████▏     | 21/50 [03:17<04:32,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 21 | Time: 0m 9s\n","\tTrain Loss: 1.577 | Train PPL:   4.838\n","\t Val. Loss: 2.448 |  Val. PPL:  11.568\n"]},{"output_type":"stream","name":"stderr","text":["\r 44%|████▍     | 22/50 [03:26<04:22,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 22 | Time: 0m 9s\n","\tTrain Loss: 1.541 | Train PPL:   4.671\n","\t Val. Loss: 2.543 |  Val. PPL:  12.719\n"]},{"output_type":"stream","name":"stderr","text":["\r 46%|████▌     | 23/50 [03:36<04:13,  9.37s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 23 | Time: 0m 9s\n","\tTrain Loss: 1.511 | Train PPL:   4.534\n","\t Val. Loss: 2.503 |  Val. PPL:  12.224\n"]},{"output_type":"stream","name":"stderr","text":["\r 48%|████▊     | 24/50 [03:45<04:03,  9.36s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 24 | Time: 0m 9s\n","\tTrain Loss: 1.482 | Train PPL:   4.403\n","\t Val. Loss: 2.553 |  Val. PPL:  12.841\n"]},{"output_type":"stream","name":"stderr","text":["\r 50%|█████     | 25/50 [03:54<03:54,  9.37s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 25 | Time: 0m 9s\n","\tTrain Loss: 1.454 | Train PPL:   4.282\n","\t Val. Loss: 2.557 |  Val. PPL:  12.898\n"]},{"output_type":"stream","name":"stderr","text":["\r 52%|█████▏    | 26/50 [04:04<03:45,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 26 | Time: 0m 9s\n","\tTrain Loss: 1.426 | Train PPL:   4.161\n","\t Val. Loss: 2.536 |  Val. PPL:  12.628\n"]},{"output_type":"stream","name":"stderr","text":["\r 54%|█████▍    | 27/50 [04:13<03:36,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 27 | Time: 0m 9s\n","\tTrain Loss: 1.403 | Train PPL:   4.069\n","\t Val. Loss: 2.529 |  Val. PPL:  12.536\n"]},{"output_type":"stream","name":"stderr","text":["\r 56%|█████▌    | 28/50 [04:23<03:26,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 28 | Time: 0m 9s\n","\tTrain Loss: 1.387 | Train PPL:   4.001\n","\t Val. Loss: 2.510 |  Val. PPL:  12.310\n"]},{"output_type":"stream","name":"stderr","text":["\r 58%|█████▊    | 29/50 [04:32<03:16,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 29 | Time: 0m 9s\n","\tTrain Loss: 1.357 | Train PPL:   3.885\n","\t Val. Loss: 2.583 |  Val. PPL:  13.230\n"]},{"output_type":"stream","name":"stderr","text":["\r 60%|██████    | 30/50 [04:41<03:07,  9.37s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 30 | Time: 0m 9s\n","\tTrain Loss: 1.335 | Train PPL:   3.799\n","\t Val. Loss: 2.672 |  Val. PPL:  14.467\n"]},{"output_type":"stream","name":"stderr","text":["\r 62%|██████▏   | 31/50 [04:51<02:58,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 31 | Time: 0m 9s\n","\tTrain Loss: 1.315 | Train PPL:   3.726\n","\t Val. Loss: 2.666 |  Val. PPL:  14.380\n"]},{"output_type":"stream","name":"stderr","text":["\r 64%|██████▍   | 32/50 [05:00<02:48,  9.38s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 32 | Time: 0m 9s\n","\tTrain Loss: 1.297 | Train PPL:   3.660\n","\t Val. Loss: 2.711 |  Val. PPL:  15.052\n"]},{"output_type":"stream","name":"stderr","text":["\r 66%|██████▌   | 33/50 [05:10<02:39,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 33 | Time: 0m 9s\n","\tTrain Loss: 1.280 | Train PPL:   3.598\n","\t Val. Loss: 2.715 |  Val. PPL:  15.098\n"]},{"output_type":"stream","name":"stderr","text":["\r 68%|██████▊   | 34/50 [05:19<02:30,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 34 | Time: 0m 9s\n","\tTrain Loss: 1.261 | Train PPL:   3.530\n","\t Val. Loss: 2.718 |  Val. PPL:  15.145\n"]},{"output_type":"stream","name":"stderr","text":["\r 70%|███████   | 35/50 [05:28<02:20,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 35 | Time: 0m 9s\n","\tTrain Loss: 1.247 | Train PPL:   3.480\n","\t Val. Loss: 2.737 |  Val. PPL:  15.442\n"]},{"output_type":"stream","name":"stderr","text":["\r 72%|███████▏  | 36/50 [05:38<02:11,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 36 | Time: 0m 9s\n","\tTrain Loss: 1.230 | Train PPL:   3.420\n","\t Val. Loss: 2.667 |  Val. PPL:  14.390\n"]},{"output_type":"stream","name":"stderr","text":["\r 74%|███████▍  | 37/50 [05:47<02:02,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 37 | Time: 0m 9s\n","\tTrain Loss: 1.213 | Train PPL:   3.362\n","\t Val. Loss: 2.681 |  Val. PPL:  14.596\n"]},{"output_type":"stream","name":"stderr","text":["\r 76%|███████▌  | 38/50 [05:57<01:52,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 38 | Time: 0m 9s\n","\tTrain Loss: 1.201 | Train PPL:   3.324\n","\t Val. Loss: 2.672 |  Val. PPL:  14.466\n"]},{"output_type":"stream","name":"stderr","text":["\r 78%|███████▊  | 39/50 [06:06<01:43,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 39 | Time: 0m 9s\n","\tTrain Loss: 1.182 | Train PPL:   3.262\n","\t Val. Loss: 2.680 |  Val. PPL:  14.580\n"]},{"output_type":"stream","name":"stderr","text":["\r 80%|████████  | 40/50 [06:15<01:33,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 40 | Time: 0m 9s\n","\tTrain Loss: 1.167 | Train PPL:   3.213\n","\t Val. Loss: 2.604 |  Val. PPL:  13.522\n"]},{"output_type":"stream","name":"stderr","text":["\r 82%|████████▏ | 41/50 [06:25<01:24,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 41 | Time: 0m 9s\n","\tTrain Loss: 1.151 | Train PPL:   3.162\n","\t Val. Loss: 2.614 |  Val. PPL:  13.650\n"]},{"output_type":"stream","name":"stderr","text":["\r 84%|████████▍ | 42/50 [06:34<01:15,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 42 | Time: 0m 9s\n","\tTrain Loss: 1.139 | Train PPL:   3.125\n","\t Val. Loss: 2.677 |  Val. PPL:  14.538\n"]},{"output_type":"stream","name":"stderr","text":["\r 86%|████████▌ | 43/50 [06:44<01:05,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 43 | Time: 0m 9s\n","\tTrain Loss: 1.125 | Train PPL:   3.080\n","\t Val. Loss: 2.663 |  Val. PPL:  14.339\n"]},{"output_type":"stream","name":"stderr","text":["\r 88%|████████▊ | 44/50 [06:53<00:56,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 44 | Time: 0m 9s\n","\tTrain Loss: 1.113 | Train PPL:   3.044\n","\t Val. Loss: 2.740 |  Val. PPL:  15.492\n"]},{"output_type":"stream","name":"stderr","text":["\r 90%|█████████ | 45/50 [07:02<00:46,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 45 | Time: 0m 9s\n","\tTrain Loss: 1.097 | Train PPL:   2.996\n","\t Val. Loss: 2.779 |  Val. PPL:  16.101\n"]},{"output_type":"stream","name":"stderr","text":["\r 92%|█████████▏| 46/50 [07:12<00:37,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 46 | Time: 0m 9s\n","\tTrain Loss: 1.086 | Train PPL:   2.962\n","\t Val. Loss: 2.757 |  Val. PPL:  15.760\n"]},{"output_type":"stream","name":"stderr","text":["\r 94%|█████████▍| 47/50 [07:21<00:28,  9.41s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 47 | Time: 0m 9s\n","\tTrain Loss: 1.074 | Train PPL:   2.926\n","\t Val. Loss: 2.773 |  Val. PPL:  16.014\n"]},{"output_type":"stream","name":"stderr","text":["\r 96%|█████████▌| 48/50 [07:31<00:18,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 48 | Time: 0m 9s\n","\tTrain Loss: 1.063 | Train PPL:   2.895\n","\t Val. Loss: 2.802 |  Val. PPL:  16.479\n"]},{"output_type":"stream","name":"stderr","text":["\r 98%|█████████▊| 49/50 [07:40<00:09,  9.39s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 49 | Time: 0m 9s\n","\tTrain Loss: 1.043 | Train PPL:   2.838\n","\t Val. Loss: 2.787 |  Val. PPL:  16.233\n"]},{"output_type":"stream","name":"stderr","text":["100%|██████████| 50/50 [07:49<00:00,  9.40s/it]"]},{"output_type":"stream","name":"stdout","text":["Epoch: 50 | Time: 0m 9s\n","\tTrain Loss: 1.035 | Train PPL:   2.815\n","\t Val. Loss: 2.777 |  Val. PPL:  16.071\n"]},{"output_type":"stream","name":"stderr","text":["\n"]}],"source":["best_valid_loss = float('inf')\n","\n","train_losses = []\n","test_losses = []\n","\n","test_phoneme_accs = []\n","\n","epochs = []\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n","    \n","    start_time = time.time()\n","    \n","    train_loss = train(model, train_loader, optimizer, vocab_criterion, mapping_criterion, CLIP)\n","    metrics = evaluate(model, eval_loader, vocab_criterion, mapping_criterion)\n","    valid_loss = metrics['vocab_loss']\n","    \n","    end_time = time.time()\n","    \n","    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n","    \n","    if valid_loss < best_valid_loss:\n","        best_valid_loss = valid_loss\n","\n","    # keep track of numbers we want to plot\n","    train_losses.append(train_loss)\n","    test_losses.append(valid_loss)\n","    test_phoneme_accs.append(metrics['phoneme_acc'])\n","    epochs.append(epoch)\n","\n","    \n","    \n","    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n","    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n","    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"]},{"cell_type":"markdown","source":["## Model Evaluation\n","Metrics to understand the model performance:\n","* Phoneme BLEU score\n","* Phoneme prediction accuracy\n","* Phoneme Word Mapping accuracy\n","* Epoch vs Loss Plot"],"metadata":{"id":"f6KdcHls9kur"}},{"cell_type":"code","source":["test_dataset = build_glove_dataset(test_pairs)\n","test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE)\n","evaluate(model, test_loader, vocab_criterion, mapping_criterion)"],"metadata":{"id":"ZvtfINjMqd3J","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1639666808643,"user_tz":300,"elapsed":5941,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"f131cab9-6e78-41bf-ef12-833771c54aab"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'mapping_acc': 0.007396738596373846,\n"," 'mapping_ed': 0.007499380040343583,\n"," 'phoneme_acc': 0.3603285350114954,\n"," 'phoneme_bleu': 0.28684838332033785,\n"," 'phoneme_edit': 0.3611303711628759,\n"," 'vocab_loss': 2.7769876207624162}"]},"metadata":{},"execution_count":37}]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","\n","plt.plot(epochs, train_losses, 'g', label='Training loss')\n","plt.plot(epochs, test_losses, 'b', label='validation loss')\n","plt.title('Training and Validation loss')\n","plt.xlabel('Epochs')\n","plt.ylabel('Loss')\n","plt.legend()\n","plt.savefig(WORKSPACE_PATH + '/transformer_train_val.pdf')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":295},"id":"-s7z5EcwEAQH","executionInfo":{"status":"ok","timestamp":1639666155003,"user_tz":300,"elapsed":650,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"9dbd7d4e-6000-43e1-98df-d2db1f9abf7a"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgUVdb48e/JQkIW1iQsCZCwyWYIEBYF2URFwV0HfFEEF4TxN+o4o4M6KvqO7zgzvoq4vjiO4ogjiqLihgsgIMomu6xCkLAmLFkIBJKc3x9VCU1IQiDpdJI+n+epp6urqqtPNaFO3Xur7hVVxRhjjP8K8HUAxhhjfMsSgTHG+DlLBMYY4+csERhjjJ+zRGCMMX7OEoExxvg5SwSmUonIFyJya2Vv60sikiIiQ7yw3/kicoc7P0pEvirPtufwPS1FJFtEAs811jL2rSLStrL3a6qWJQKDe5IonApE5KjH+1Fnsy9VvVxVp1X2ttWRiEwUkQUlLI8SkeMi0qW8+1LV6ap6aSXFdUriUtVfVTVCVfMrY/+m9rFEYHBPEhGqGgH8ClzpsWx64XYiEuS7KKult4ELRSSh2PKRwFpVXeeDmIw5a5YITKlEZKCIpIrIn0RkL/CGiDQUkU9FJE1EDrnzcR6f8azuGCMii0TkGXfb7SJy+TlumyAiC0QkS0S+EZGXROTtUuIuT4z/LSLfu/v7SkSiPNbfIiI7ROSAiDxS2u+jqqnAXOCWYqtGA2+dKY5iMY8RkUUe7y8RkY0ikiEiLwLisa6NiMx140sXkeki0sBd92+gJTDbLdE9KCLxbhVOkLtNcxH5REQOishWEbnTY9+TROQ9EXnL/W3Wi0hyab9BsWOo734uzf39/iwiAe66tiLynXs86SIyw10uIvKciOwXkUwRWXs2JSlTOSwRmDNpCjQCWgHjcP5m3nDftwSOAi+W8fnewCYgCvg78LqIyDls+w6wFGgMTOL0k6+n8sT4X8BYIAaoA/wRQEQ6Aa+4+2/ufl+JJ2/XNM9YROQ8IMmN92x/q8J9RAEfAn/G+S1+Afp6bgL81Y2vI9AC5zdBVW/h1FLd30v4ineBVPfzNwD/IyKDPdZf5W7TAPikPDG7XgDqA62BATgJcay77r+Br4CGOL/nC+7yS4H+QHv3s78BDpTz+0xlUVWbbCqagBRgiDs/EDgOhJaxfRJwyOP9fOAOd34MsNVjXRigQNOz2RbnJJoHhHmsfxt4u5zHVFKMf/Z4/1vgS3f+MeBdj3Xh7m8wpJR9hwGZwIXu+6eAj8/xt1rkzo8GfvTYTnBO3HeUst9rgJUl/Ru67+Pd3zIIJ2nkA5Ee6/8KvOnOTwK+8VjXCThaxm+rQFsg0P2dOnmsuwuY786/BUwF4op9fjCwGegDBPj6799fJysRmDNJU9VjhW9EJExE/s8t+mcCC4AGUvodKXsLZ1Q1x52NOMttmwMHPZYB7Cwt4HLGuNdjPscjpuae+1bVI5RxherG9D4w2i29jMI56Z3Lb1WoeAzq+V5EmojIuyKyy93v2zglh/Io/C2zPJbtAGI93hf/bULlzO1DUUCwu6+S9vsgTkJb6lY33eYe21ycEsdLwH4RmSoi9cp5LKaSWCIwZ1K8e9o/AOcBvVW1Hk6xHjzqsL1gD9BIRMI8lrUoY/uKxLjHc9/udzY+w2em4VRpXAJEArMrGEfxGIRTj/d/cP5dznf3e3OxfZbVpfBunN8y0mNZS2DXGWI6k3TgBE412Gn7VdW9qnqnqjbHKSm8LO5tp6o6RVV74JQ+2gMPVDAWc5YsEZizFYlT131YRBoBj3v7C1V1B7AcmCQidUTkAuBKL8U4ExguIv1EpA7wJGf+f7IQOIxT9fGuqh6vYByfAZ1F5Dr3SvwenCqyQpFANpAhIrGcfuLch1NPfxpV3QksBv4qIqEikgjcjlOqOGfq3Jr6HvCUiESKSCvg/sL9isiNHg3lh3CSVYGI9BSR3iISDBwBjgEFFYnFnD1LBOZsTQbq4lwB/gh8WUXfOwq4AKea5i/ADCC3lG3POUZVXQ/cjdPYuwfnpJV6hs8oTnVQK/e1QnGoajpwI/A0zvG2A7732OQJoDuQgZM0Piy2i78CfxaRwyLyxxK+4iacdoPdwCzgcVX9pjyxncHvcE7m24BFOL/hv9x1PYElIpKN0wB9r6puA+oBr+H8zjtwjvcflRCLOQviNtgYU6O4tx9uVFWvl0iMqe2sRGBqBLcKoY2IBIjIUOBq4CNfx2VMbWBPipqaoilOFUhjnKqaCaq60rchGVM7WNWQMcb4OasaMsYYP1fjqoaioqI0Pj7e12EYY0yNsmLFinRVjS5pXY1LBPHx8SxfvtzXYRhjTI0iIjtKW2dVQ8YY4+csERhjjJ+zRGCMMX6uxrURGGOq3okTJ0hNTeXYsWNn3tj4VGhoKHFxcQQHB5f7M5YIjDFnlJqaSmRkJPHx8ZQ+rpDxNVXlwIEDpKamkpBQfATV0lnVkDHmjI4dO0bjxo0tCVRzIkLjxo3PuuRmicAYUy6WBGqGc/l38ptEsG7/Oh759hEO5NhwqMYY48lvEsHWg1v5n0X/w68Zv/o6FGPMWTpw4ABJSUkkJSXRtGlTYmNji94fP368zM8uX76ce+6554zfceGFF1ZKrPPnz2f48OGVsq+q4jeNxTHhMQDsP7Lfx5EYY85W48aNWbVqFQCTJk0iIiKCP/7x5Jg7eXl5BAWVfDpLTk4mOTn5jN+xePHiygm2BvKbEkF0mNPFRlpOmo8jMcZUhjFjxjB+/Hh69+7Ngw8+yNKlS7ngggvo1q0bF154IZs2bQJOvUKfNGkSt912GwMHDqR169ZMmTKlaH8RERFF2w8cOJAbbriBDh06MGrUKAp7af7888/p0KEDPXr04J577jnjlf/Bgwe55pprSExMpE+fPqxZswaA7777rqhE061bN7KystizZw/9+/cnKSmJLl26sHDhwkr/zUpjJQJjzFm578v7WLV3VaXuM6lpEpOHTj7rz6WmprJ48WICAwPJzMxk4cKFBAUF8c033/Dwww/zwQcfnPaZjRs3Mm/ePLKysjjvvPOYMGHCaffcr1y5kvXr19O8eXP69u3L999/T3JyMnfddRcLFiwgISGBm2666YzxPf7443Tr1o2PPvqIuXPnMnr0aFatWsUzzzzDSy+9RN++fcnOziY0NJSpU6dy2WWX8cgjj5Cfn09OTs5Z/x7nym8SQb2QetQJrGOJwJha5MYbbyQwMBCAjIwMbr31VrZs2YKIcOLEiRI/M2zYMEJCQggJCSEmJoZ9+/YRFxd3yja9evUqWpaUlERKSgoRERG0bt266P78m266ialTp5YZ36JFi4qS0eDBgzlw4ACZmZn07duX+++/n1GjRnHdddcRFxdHz549ue222zhx4gTXXHMNSUlJFfptzobfJAIRITosmrQjVjVkTEWcy5W7t4SHhxfNP/roowwaNIhZs2aRkpLCwIEDS/xMSEhI0XxgYCB5eXnntE1FTJw4kWHDhvH555/Tt29f5syZQ//+/VmwYAGfffYZY8aM4f7772f06NGV+r2l8Zs2AnCqh/bnWInAmNooIyOD2NhYAN58881K3/95553Htm3bSElJAWDGjBln/MxFF13E9OnTAaftISoqinr16vHLL79w/vnn86c//YmePXuyceNGduzYQZMmTbjzzju54447+Omnnyr9GErjf4nAqoaMqZUefPBBHnroIbp161bpV/AAdevW5eWXX2bo0KH06NGDyMhI6tevX+ZnJk2axIoVK0hMTGTixIlMmzYNgMmTJ9OlSxcSExMJDg7m8ssvZ/78+XTt2pVu3boxY8YM7r333ko/htLUuDGLk5OT9VwHphk9azQLf13I9nu3V3JUxtRuGzZsoGPHjr4Ow+eys7OJiIhAVbn77rtp164dv//9730d1mlK+vcSkRWqWuJ9tF4tEYhIioisFZFVInLa2VtEBopIhrt+lYg85s14rI3AGFMRr732GklJSXTu3JmMjAzuuusuX4dUKaqisXiQqqaXsX6hqlbJY3gx4TEcOXGEI8ePEF4n/MwfMMYYD7///e+rZQmgovyujQDsoTJjjPHk7USgwFciskJExpWyzQUislpEvhCRziVtICLjRGS5iCxPSzv3k3h0uPt0sVUPGWNMEW9XDfVT1V0iEgN8LSIbVXWBx/qfgFaqmi0iVwAfAe2K70RVpwJTwWksPtdg7OliY4w5nVdLBKq6y33dD8wCehVbn6mq2e7850CwiER5Kx5LBMYYczqvJQIRCReRyMJ54FJgXbFtmoo7ioKI9HLj8dqAAYUdz1kiMKb2K+xEbvfu3dxwww0lbjNw4EDOdDv65MmTT+n354orruDw4cMVjm/SpEk888wzFd5PZfBmiaAJsEhEVgNLgc9U9UsRGS8i491tbgDWudtMAUaqFx9sCK8TTlhwmDUWG+NHmjdvzsyZM8/588UTweeff06DBg0qI7Rqw2uJQFW3qWpXd+qsqk+5y19V1Vfd+RfddV1VtY+qer1DcHu62JiaZ+LEibz00ktF7wuvprOzs7n44ovp3r07559/Ph9//PFpn01JSaFLly4AHD16lJEjR9KxY0euvfZajh49WrTdhAkTSE5OpnPnzjz++OMATJkyhd27dzNo0CAGDRoEQHx8POnpzh3xzz77LF26dKFLly5Mnjy56Ps6duzInXfeSefOnbn00ktP+Z6SrFq1ij59+pCYmMi1117LoUOHir6/U6dOJCYmMnLkSKDkLqwrym86nStkicCYirnvPlhVub1Qk5QEk8voy27EiBHcd9993H333QC89957zJkzh9DQUGbNmkW9evVIT0+nT58+XHXVVaWO2/vKK68QFhbGhg0bWLNmDd27dy9a99RTT9GoUSPy8/O5+OKLWbNmDffccw/PPvss8+bNIyrq1ObLFStW8MYbb7BkyRJUld69ezNgwAAaNmzIli1b+M9//sNrr73Gb37zGz744ANuvvnmUo9v9OjRvPDCCwwYMIDHHnuMJ554gsmTJ/P000+zfft2QkJCiqqjSurCuqL86jkCcNoJLBEYU7N069aN/fv3s3v3blavXk3Dhg1p0aIFqsrDDz9MYmIiQ4YMYdeuXezbt6/U/SxYsKDohJyYmEhiYmLRuvfee4/u3bvTrVs31q9fz88//1xmTIsWLeLaa68lPDyciIgIrrvuuqLBZBISEoq6ke7Ro0dRR3UlycjI4PDhwwwYMACAW2+9lQULFhTFOGrUKN5+++2iEdgKu7CeMmUKhw8fLnVktrPhlyWC1ftW+zoMY2qssq7cvenGG29k5syZ7N27lxEjRgAwffp00tLSWLFiBcHBwcTHx3Ps2LGz3vf27dt55plnWLZsGQ0bNmTMmDHntJ9CxbuxPlPVUGk+++wzFixYwOzZs3nqqadYu3ZtiV1Yd+jQ4ZxjBT8sERRWDdW0zvaM8XcjRozg3XffZebMmdx4442AczUdExNDcHAw8+bNY8eOHWXuo3///rzzzjsArFu3rmjoyMzMTMLDw6lfvz779u3jiy++KPpMZGRkifXwF110ER999BE5OTkcOXKEWbNmcdFFF531cdWvX5+GDRsWlSb+/e9/M2DAAAoKCti5cyeDBg3ib3/7GxkZGWRnZ5fYhXVF+WWJ4Hj+cTJzM6kfWnYXssaY6qNz585kZWURGxtLs2bNABg1ahRXXnkl559/PsnJyWe8Mp4wYQJjx46lY8eOdOzYkR49egAUdf/coUMHWrRoQd++fYs+M27cOIYOHUrz5s2ZN29e0fLu3bszZswYevVyHo+644476NatW5nVQKWZNm0a48ePJycnh9atW/PGG2+Qn5/PzTffTEZGBqrKPffcQ4MGDXj00UeZN28eAQEBdO7cmcsvv/ysv684v+qGGuDfq//N6I9Gs+V3W2jbqG0lRmZM7WXdUNcs1aob6urIni42xphTWSIwxhg/53eJoLAHUksExpydmlaN7K/O5d/J/xJBmHVFbczZCg0N5cCBA5YMqjlV5cCBA2f9kJnf3TUUEhRC/ZD6ViIw5izExcWRmppKRcYDMVUjNDSUuLi4s/qM3yUCcJ8lyLFEYEx5BQcHk5CQ4OswjJf4XdUQOO0EViIwxhiHXyaCmPAYayMwxhiXfyaCMOuB1BhjCvllIogOjyY9J50CLfB1KMYY43NeTQQikiIia0VklYic1i+EOKaIyFYRWSMi3UvaT2WLCY8hX/M5dPRQVXydMcZUa1Vx19AgVU0vZd3lQDt36g284r56lefTxY3DGnv764wxplrzddXQ1cBb6vgRaCAizbz9pdbNhDHGnOTtRKDAVyKyQkTGlbA+Ftjp8T7VXXYKERknIstFZHllPNBS+HSxJQJjjPF+Iuinqt1xqoDuFpH+57ITVZ2qqsmqmhwdHV3hoApLBGk5dgupMcZ4NRGo6i73dT8wC+hVbJNdQAuP93HuMq9qHNYYQaxEYIwxeDERiEi4iEQWzgOXAuuKbfYJMNq9e6gPkKGqe7wVU6GggCAahzW2RGCMMXj3rqEmwCwRKfyed1T1SxEZD6CqrwKfA1cAW4EcYKwX4zlFdFi0VQ0ZYwxeTASqug3oWsLyVz3mFbjbWzGUpXAQe2OM8Xe+vn3UZywRGGOMw28TQXSY9UBqjDHgx4kgJjyGg0cPkleQ5+tQjDHGp/w6EQCk55TW+4UxxvgHv08EVj1kjPF3fpsIosOtmwljjAE/TgRF3UzYSGXGGD/n94nASgTGGH/nt4mgQWgDAiXQEoExxu/5bSIIkACiw62bCWOM8dtEAPZ0sTHGgCUCSwTGGL/n14nAupkwxhg/TwQx4THWRmCM8Xt+nwgyczM5lnfM16EYY4zP+HUiKBzE3h4qM8b4M68nAhEJFJGVIvJpCevGiEiaiKxypzu8HY8nG8TeGGO8O1RloXuBDUC9UtbPUNX/VwVxnMaeLjbGGC+XCEQkDhgG/NOb33OuLBEYY4z3q4YmAw8CBWVsc72IrBGRmSLSoqQNRGSciCwXkeVpaZVXjWM9kBpjjBcTgYgMB/ar6ooyNpsNxKtqIvA1MK2kjVR1qqomq2pydHR0pcUYWSeSkMAQayw2xvg1b5YI+gJXiUgK8C4wWETe9txAVQ+oaq779p9ADy/GcxoRcZ4uzrESgTHGf3ktEajqQ6oap6rxwEhgrqre7LmNiDTzeHsVTqNylYoOt6eLjTH+rSruGjqFiDwJLFfVT4B7ROQqIA84CIyp6nisvyFjjL+rkkSgqvOB+e78Yx7LHwIeqooYShMTHsOGtCoviBhjTLXh108WA8SEOSUCVfV1KMYY4xN+nwiiw6M5mneUIyeO+DoUY4zxCb9PBDaIvTHG31kisKeLjTF+zu8TQWEPpJYIjDH+yu8TQfPI5gBsP7zdx5EYY4xv+H0iiK0XS0KDBOZun+vrUIwxxif8PhEADGk9hHkp88gryPN1KMYYU+UsEQCXtL6EzNxMlu9e7utQjDGmylkiAAYnDEYQvv7la1+HYowxVc4SAdA4rDHdm3Xnm+3f+DoUY4ypcpYIXENaD+GHnT+QfTzb16EYY0yVskTgGtJ6CCcKTrBgxwJfh2KMMVXKEoGrX8t+hAaF8s02qx4yxvgXSwSu0KBQ+rXsx9fbrMHYGONfvJ4IRCRQRFaKyKclrAsRkRkislVElohIvLfjKcslrS9h3f517M3e68swjDGmSlVFieBeSh+C8nbgkKq2BZ4D/lYF8ZRqSOshAHy77VtfhmGMMVXKq4lAROKAYTgD05fkamCaOz8TuFhExJsxlSWpaRKN6za26iFjjF/xdolgMvAgUFDK+lhgJ4Cq5gEZQOPiG4nIOBFZLiLL09K8N25AgARwceuL+WbbNzZimTHGb3gtEYjIcGC/qq6o6L5UdaqqJqtqcnR0dCVEV7ohCUPYlbWLTQc2efV7jDGmuvBmiaAvcJWIpADvAoNF5O1i2+wCWgCISBBQHzjgxZjOqLCdwLqbMMb4C68lAlV9SFXjVDUeGAnMVdWbi232CXCrO3+Du41P62QSGibQpmEb627CGOM3qvw5AhF5UkSuct++DjQWka3A/cDEqo6nJENaD2He9nmcyD/h61CMMcbrypUIRCRcRALc+fYicpWIBJf3S1R1vqoOd+cfU9VP3PljqnqjqrZV1V6quu1cDqKyXdL6ErKOZ7Fs9zJfh2KMMV5X3hLBAiBURGKBr4BbgDe9FZSvDUoYhCDW3YQxxi+UNxGIquYA1wEvq+qNQGfvheVbjeo2okfzHvY8gTHGL5Q7EYjIBcAo4DN3WaB3QqoeLml9CT+m/khWbpavQzHGGK8qbyK4D3gImKWq60WkNTDPe2H53pDWQ8gryLNuqY0xtV65EoGqfqeqV6nq39xG43RVvcfLsflU3xZ9qRtUl482fuTrUIwxxqvKe9fQOyJST0TCgXXAzyLygHdD862QoBBu7Xor01ZPI+Vwiq/DMcYYrylv1VAnVc0ErgG+ABJw7hyq1R7p/wgBEsCT3z3p61CMMcZrypsIgt3nBq4BPlHVE0Ct75Utrl4cv+35W6atnsamdOt7yBhTO5U3EfwfkAKEAwtEpBWQ6a2gqpOJ/SZSN6guj89/3NehGGOMV5S3sXiKqsaq6hXq2AEM8nJs1UJMeAz39r6XGetnsHrval+HY4wxla68jcX1ReTZwjEBROR/cUoHfuGPF/6R+iH1eWz+Y74OxRhjKl15q4b+BWQBv3GnTOANbwVV3TSs25AHLnyATzZ9wpLUJb4OxxhjKlV5E0EbVX1cVbe50xNAa28GVt3c2+deosKi+PO8P/s6FGOMqVTlTQRHRaRf4RsR6Qsc9U5I1VNEnQge6vcQ32z7hvkp830djjHGVJryJoLxwEsikuKOOPYicJfXoqqmJiRPoHlkcx6Z+4iNaWyMqTXKe9fQalXtCiQCiaraDRjs1ciqobrBdXm0/6Ms3rmYL7d+6etwjDGmUpzVCGWqmuk+YQzOiGKlEpFQEVkqIqtFZL2IPFHCNmNEJE1EVrnTHWcTjy/c1u024hvE87svfseuzF2+DscYYyqsIkNVyhnW5wKD3ZJEEjBURPqUsN0MVU1yp39WIJ4qUSewDm9f+zb7j+yn/5v92XF4h69DMsaYCqlIIiizktx98CzbfRvsTrWiYr1vy758fcvXHDx6kP5v9ueXg7/4OiRjjDlnZSYCEckSkcwSpiyg+Zl2LiKBIrIK2A98raol3YR/vYisEZGZItKilP2MK3yYLS0trTzH5XW943rz7ehvOXL8CP3f7M/G9I2+DskYY85JmYlAVSNVtV4JU6SqBp1p56qar6pJQBzQS0S6FNtkNhCvqonA18C0UvYzVVWTVTU5Ojq6fEdWBbo36878MfPJK8hjwJsDWLtvra9DMsaYs1aRqqFyU9XDOCOaDS22/ICq5rpv/wn0qIp4KlOXmC58N+Y7ggKCGDhtID/t+cnXIRljzFnxWiIQkWgRaeDO1wUuATYW26aZx9urgA3eisebOkR1YMGYBUTUiWDQtEF8seULX4dkjDHl5s0SQTNgnoisAZbhtBF8KiJPishV7jb3uLeWrgbuAcZ4MR6vatOoDQvHLiShQQLD3hnGP77/hz10ZoypEaSmnaySk5N1+fLlvg6jVEeOH2Hsx2N5/+f3GXX+KF678jXqBtf1dVjGGD8nIitUNbmkdVXSRuBPwuuEM+OGGfxl0F+YvnY6/d/sT2pmqq/DMsaYUlki8AIR4ZH+j/DxyI/ZmL6R5KnJLN652NdhGWNMiSwReNFV513Fj7f/WNSI/PpPr/s6JGOMOY0lAi/rHNOZpXcuZUCrAdwx+w7u/uxujucf93VYxhhTxBJBFWhUtxGfj/qcBy58gJeXv8yQt4awL3ufr8MyxhjAEkGVCQoI4u+X/J13rnuH5buXk/xaMst2LfN1WMYYY4mgqt10/k0svn0xgRLIRW9cxFur3/J1SMYYP2eJwAeSmiaxfNxyLmxxIbd+dCs3vHcDmw9s9nVYxhg/ZYnAR6LCovjqlq94cuCTfLn1Szq91InffvZb9mbv9XVoxhg/Y4nAh4ICgnh0wKP8cs8vjE8ez2s/vUabKW14bN5jZOZmnnkHxhhTCSwRVANNIprw4hUvsuHuDVzZ/kr+e8F/03ZKW55Z/AwZxzJ8HZ4xppazRFCNtG3UlndveJdldy6ja9OuPPD1A7R4rgX3z7nfhsQ0xniNdTpXjf205yee/eFZZqyfgapyfafr+cMFf6Bn857syNjBmn1rWL13NWv2r2HNvjVkH89mYt+JjE8eT3BgsK/DN8ZUI2V1OmeJoAZIzUxlypIpTF0xlYzcDMKCw8g5kVO0vk3DNiQ2SeTg0YN8t+M7OkR14NlLn+Xydpf7MGpjTHViiaCWyMrN4o1Vb7D14Fa6xHQhsUkiXWK6EFEnAgBVZfbm2fzhqz+w9eBWhrYdyv9e+r90iu7k48iNMb7mk0QgIqHAAiAECAJmqurjxbYJAd7CGaLyADBCVVPK2q8/J4LyOp5/nBeXvsiT3z1J9vFsxiePZ9LASUSFRfk6NGOMj/hqPIJcYLCqdgWSgKEi0qfYNrcDh1S1LfAc8DcvxuM36gTW4f4L7mfL77ZwV4+7eGX5K7Sd0pZ/fP8PcvNyz7wDY4xf8VoiUEe2+zbYnYoXP64GprnzM4GLRUS8Ec8vv8CkSXDihDf2Xj1Fh0fz0rCXWDN+DRe2uJAHv3mQji915L3179kwmsaYIkHe3LmIBAIrgLbAS6q6pNgmscBOAFXNE5EMoDGQXtmxrFsHTzwBffvCJZdU9t6rt84xnfl81Od8/cvX/OGrPzBi5ggmx03m2cuepVN0J3Zn7WZX5i52Ze1id9ZudmftpnXD1oxNGkv90Pq+Dt8Y42VV0lgsIg2AWcDvVHWdx/J1wFBVTXXf/wL0VtX0Yp8fB4wDaNmyZY8dO87+nvqjRyEqCm69FV5++dyPpabLL8jnzVVv8ud5fy61O4vIOpFkHc8iok4EY5PGck/ve2jbqG0VR2qMqUzV4q4hEXkMyFHVZzyWzQEmqeoPIhIE7AWitYygKtJYfNqBhOMAABrNSURBVMMNsHgxpKZCgJ8/Spd9PJvXf3qd4/nHia0XS/PI5sRGOq/hdcL5ac9PPL/kef6z9j/kFeQxvP1w7utzH4PiB+Gl2jtjjBf56q6haOCEqh4WkbrAV8DfVPVTj23uBs5X1fEiMhK4TlV/U9Z+K5IIpk+Hm292ksEFF5zTLvzOnqw9vLL8FV5Z/grpOen0aNaDl4e9TK/YXr4OzRhzFnx111AzYJ6IrAGWAV+r6qci8qSIXOVu8zrQWES2AvcDE70YD8OGQVAQzJrlzW+pXZpFNuPJQU+y8/c7ee3K19iTvYc+/+zD3Z/dzeFjh30dnjGmEvjdA2VDh8LWrbBlC1gNx9nLzM3k0bmP8uKyF4kJj+G5y55jROcRVl1kTDXnqxJBtXTttc6tpOvWnXlbc7p6IfV4/vLnWXrHUmIjY7npg5sYOn0oWw9u9XVoxphz5HeJ4OqrnZKAVQ9VTI/mPVhyxxKmDJ3CDzt/oNNLnRgxcwRzt8+1ZxSMqWH8rmoIoF8/OHIEVq6spKD83O6s3fz9+7/z1uq3OHTsEO0atWNcj3Hc2vVWosOjfR2eMQarGjrNtdfCqlWwfbuvI6kdmkc2Z/LQyey6fxdvXfMWTSKa8MDXDxD3XBw3fXATX279kryCPF+HaYwphd8mArDqocpWN7gut3S9hYVjF7JuwjomJE9gztY5XD79cmKfjeW+L+9j2a5lVnVkTDXjl1VDAElJEBkJCxdWQlCmVLl5uXy+5XOmr53O7M2zOZ5/nPaN2zPq/FH0iu1FdFg00eHRRIdFUze4rq/DNabWqhZPFleWykoETzzhTHv2QJMmlRCYOaPDxw7zwc8f8Pbat/ku5Tu0WB+EEXUiiA6LpnXD1vRs3pOesT3p2bwncfXi7PZUYyrIEkEJ1qyBrl3h//4Pxo2rhMDMWdmbvZdth7aRdiSN/Uf2k5aT5szn7GdT+iZW71td1K7QJLwJPWN7clHLi7g58WaaRzb3cfTG1DyWCEqgCu3aOdMXX1RCYKZSHcs7xuq9q1m2exnLdi9j6a6lbEzfSKAEMqz9MO7sfidD2w4lKOD0DnRz83JZuXclG9I2cEW7K2gSYUU+Y8pKBF7thro6E3EajZ9/HjIyoL71tlythAaF0juuN73jehct23JgC6+vfJ03V73JJ5s+ITYyltu63cZ1Ha9jy4Et/JD6Az+k/sBPe37ieP5xAJpGNOW9G97jolYX+epQjKn2/LZEAE7nc337Op3R/dd/VcouTRU4kX+C2Ztn89pPrzFn65yitobQoFCSmydzQdwF9InrQ3RYNLd/cjvbDm3j6SFP84cL/mBtDabG2bsXli2D5cuhTx+4/PJz249VDZWioABiY50HzN5/v1J2aarYrxm/Mj9lPp2iO9G1SVeCA4NPWZ+Zm8nYj8fy4YYPua7jdfzrqn/ZYDum0i1bBvv3w4UXQsOGZW979CgsXep0cxMUBHXqQEjIyVdVWLv25Ml/1y7ncwEB8Mgj8OST5xajJYIyTJgAb70F6elQ1+5erJVUled+fI4Hv36QhIYJfPCbD0hskujrsEw1lJbmnA/mz4e77oLhw8vePi8PHnsM/vpX570InH8+XHTRySkyEn74ARYsgO++c5LA8eNnjuW88yA52Zl69nRueQ8PP/djs0RQhq++gssug9degzvuqLTdmmpo4Y6FjJg5gsPHDnN9p+tpENKAeiH1iqb6ofXpHN2Zrk27+jpUU4UKCmDePOcc8OGHzrjmUVHOxeFNNzntiNEl9JSSmuqsX7QIbr/dqV7+/nvnhP/DD043NuBcyRcUQGAg9OgBAwZA//7OPDhJITf35Gt+vpMEKrvd0hJBGfLyYPBg+PFH+PxzGDKk0nZtqqG92XuZ8NkEVu1dRWZuJhnHMsjX/FO2uTjhYib2m8jFCRdbm0INcPw47NwJv/7qnHDr1XOuwuvVc6bCq+gjR5wbQzIy4PBh53X1avjnP50eiRs2hNGj4c47nbsJn34a/vIX54Q8ZQqMHHmy6/ovvoBbboFjx5xb0EeNOjWmEyecbmwWLnS+p18/ZzCsiIiq/W08WSI4g0OHnAydkuIUCQsztan9VJWjeUeLksInmz7huR+fY0/2Hro3686f+v6J6zteT2BAoK9D9XtHj5684t6yBXbscKY9e5x69dIEBDgn8Pz8ktf37+88S3T99RAaeuq69eudq/0lS5xqohdecE78Tz/tVAG9/75z9V4T+GqoyhbAW0ATQIGpqvp8sW0GAh8Dhd2/faiqZTaFeCMRAOze7TT05OQ4f2zt2lX6V5gaIjcvl3+v+Tf/WPwPNh/YTJuGbbi75930jO1Jp+hONKrbyNch+oW8PKex9NtvnWnxYqfqJDAQWrUqeQoOhszM06eCAufKvn59aNDg5HxsLLRsWXYc+flOieCRR5wSgKqTOCZPrlntir5KBM2AZqr6k4hEAiuAa1T1Z49tBgJ/VNUzNMmc5K1EALB5s3M7aUSE80fXrJlXvsbUEPkF+Xy86WOeXvQ0y3YvK1reNKIpnaM70ym6E4lNEhnefjhNI5r6MNKaqaDAqYffufPk1X1Kysn5rVshO9vZtmtXuPhiZypsgK1q27bB4487Q96OHFn1319R1aJqSEQ+Bl5U1a89lg2kGiUCcG7ZGjQI2rRxWvgbNPDaV5kaQlX5NeNX1qetZ/3+9fyc/rPzmvYzR04cIUACGJwwmFHnj+K6jtdRL6Ser0P2quxs2LfPufqOiHCmOnVO3ebIEee2x927ndfCyfP9nj1OXbqniIiTV/etWzsn/UGDSm6sNWfH54lAROKBBUAXVc30WD4Q+ABIBXbjJIX1JXx+HDAOoGXLlj127Njh1Xi/+sqpD7zgApgz5/R6Q2MACrSADWkbeHfdu7yz7h22HdpGSGAIV553JaPOH8VlbS6rkT2qFhQ4jafLlzt9chU/iWdlnf6Z4GDnKj083KmKycg4fZuICKcqpqQpPt45+TdsaGOJe4tPE4GIRADfAU+p6ofF1tUDClQ1W0SuAJ5X1TJr571dIij0n/84t4N17QrPPuvcWWRMaVSVJbuWMH3NdGasn0FaThphwWEMaT2E4e2GM7z9cJpFVk5dY3o6fPABfPmlU2cNJ0+eIs7UsiUkJjp/v126lFyVogoHDjgn982bnRP/8uWwYsXJE3lQEDRv7pysPV+bNnXq8LOzT58iI0/fPjbWN9U55iSfJQIRCQY+Beao6rPl2D4FSFbV9NK2qapEAM49xfff79RXDhsGf/87dOpUJV9tarC8gjzmbp/L7E2zmb15NjsynBJsj2Y9uLL9ldzQ6QY6x3Q+q30ePuwMpDRjBnzzjdOA2bq1c7974X/hwtf8fOeKPjPz5Odbt3YSQ1iYc/97aqqTAHJzT24THOwkjsKHmJKTnb/34FMf1jY1lK8aiwWYBhxU1ftK2aYpsE9VVUR6ATOBVlpGUFWZCMC54nrhBXjqKadIfOedzjgGNoaBKQ9VZd3+dXy6+VNmb57Nj6k/oii9YntxW9JtjOwy8pQuLwoKnCqY7dtPTitWOFWUx49DQgKMGOFMXbuWXo2i6txXv2aNM61e7bwePw5xcc4UG3vyNSHBKTmEhFTRD2OqnK8SQT9gIbAWKHAXPwy0BFDVV0Xk/wETgDzgKHC/qi4ua79VnQgKpac7fXy88orTZvBf/+X85yms4yz8D1WRR8DN2VN1+niJifF93XJ+vnPhcPSoc6UdGupUh3g2pO7L3sc7a99h6uIZbNwAwQe70q7gaupl9ebAzkbs2CGndT+QkOD0lDtihNPVgK+P09RMPm8srky+SgSFNm+Ghx+GuXOdB9GKa9IELr0UrrjCeW1kt5x7RU6O047zwgvO1W5UlPP05kUXOa/dulVOlcaxY85tg9u3l3zny759TixHjzp15iUJCTn5pGtkpNOfze7dHhsE5kLURoKitxEWs496TQ7SuHkmTeJyaB53goToZtze7XZi68VW/ICM37JE4CU5Oc7JoLC+NTXV6TVwzhynES4gwOk29oornPufS2ssq18fGjeu2MMpqk6xv7YX7VNS4OWX4fXX4eBB5+nOkSOdBL1woXPSBue37NPHqedOSnKm9u2dxs/ijhxxPrd16+nTzp2nPrUq4pQ+ChtBmzVzSoF16zolgMLXkBAniWRlOXX1ha+Zmc4tyZ06OVPHjtAkLodPtnzIktQlHDp2iEPHDnH42GEOHXXm9x/ZT6AEMrrraB648AHOi6ohj7KaasUSQRXLz3eeR/jiC6f/ovKGGxbmXNlGRTmJIT7eOdF16eK8RkWd+h1r1jhPQS9a5Ey7djknp/POc0567ds7823aOCfAggJnys8/OV+aoKCTJ7TQ0JNTSSfSQjk5zom68Ap627ZTY+rQwXlt1uxk9YaqU+2WknJy8mzkhJPbrlkDs2efHFTod79zSgCeVSW7d5/8Tb7/3knMhVUtoaHOb9m168kG1a1bnfvZPUVFQdu2ztPlbds6U0KCU/3XtGnVN55uP7Sd//3hf3l95evk5uVyXcfrmNhvIsnNS/w/bUyJLBH42L59Tm+ExR+eAedknJnpnAzT052SRHq6U32wdatz1VuoSRMnIYg4neQV3s8dG+ucENu3d06kmzfDpk0lV11VVECAkwyCg52pcD4/36mr9xQW5sS2a5eTJApFRjqxHj3qxOu5Dk7eAgmnXo1HRzuN9XfdBS1alC/eEydg40anA7DVq0++BgefPMm3besky8LX6voQ4f4j+5myZAovLXuJw8cOM6DVANo2aktIYAghQSHUCaxTNN+3RV8uanURARLg67BNNWGJoIZSdUYnWrvWGcRi7Vpnystz+kXq18/pEqNly5IbEA8ccJLC9u1OwgkIODkFBp56wi0uL8+p2vCccnOd17w85wRb+HrihLPPli2d2xQTEpzX6Ghn/wUFTjLYtMmZNm504goPd0o9nlOrVjZs6Jlk5mYydcVU3lj1BhnHMsjNzyU3L5fc/NyiIToB4hvEMzpxNKO7jqZNozY+jNhUB5YIjPETqkr28Ww+2fQJ01ZP45tt36Ao/Vr249autzKg1QDC64QTFhxGWHAYdQLrnHmnplawRGCMn0rNTOXtNW8zbfU0NqZvPG19UEAQYcFhxDeIZ3yP8dzS9RYi6viw03zjNZYIjPFzqsqKPSvYmL6RnBM5HDl+xHk94bwu+nURK/asoH5IfW7rdht397zbqpNqGUsExpgyqSo/pv7IC0tf4P2f3ye/IJ/h7YczPnk8vWJ7ERUWdeadmGrNEoExptx2Z+3m1eWv8uryV0nLSQMgKiyKjlEd6RjVkU7Rnegc05n+rfpbG0MNYonAGHPWcvNymZ8yn5/TfmZD+oai14NHnXuaW9RrwQMXPsAd3e+okd1t+xtLBMaYSqGqpOWk8cPOH3jmh2dY9OsiYsJj+H2f3/Pbnr+t9YPy1GRlJQJ72sQYU24iQkx4DFd3uJqFYxeyYMwCujfrzkPfPkTL51ry6NxH2Ze9z9dhmrNkJQJjTIWt2L2Cvy76Kx9u+JAACeCKdlcwNmksw9oPs3aEasKqhowxVWLzgc38a+W/mLZ6Gnuz9xIdFs3NiTczNmks5zc539fh+TVLBMaYKpVXkMecrXN4Y9UbfLLpE04UnKBdo3Z0b9adpKZJdGvajaSmSTSJsBGeqoolAmOMz6TnpPPO2neYnzKflXtXknI4pWhds4hmJDdP5pLWl3BZ28to16gdYiPveIWvRihrAbwFNAEUmKqqzxfbRoDngSuAHGCMqv5U1n4tERhTsx06eojV+1azau8qVu5dyeKdi9l6cCvgdJR3WZvLuKzNZQxOGHzKMJ6mYnyVCJoBzVT1JxGJBFYA16jqzx7bXAH8DicR9AaeV9XeZe3XEoExtc+2Q9uYs3UOc36Zw9ztc8k6nkWABNAhqgM9mvWgR7MedG/WnW7NullfSOeoWlQNicjHwIuq+rXHsv8D5qvqf9z3m4CBqrqnlN1YIjCmljuRf4IfUn9g7va5rNizghW7V7An2zklCMJ5UefRumFr4iLjiK0XS1y9OGIjndf2jdsTHFjFIwfVEGUlgjLGm6rUAOKBbsCSYqtigZ0e71PdZackAhEZB4wDaNmypbfCNMZUA8GBwfRv1Z/+rfoXLduTtacoKazcu5JfM35l+e7l7D9y6mhIreq34rEBjzG662iCAqrk9FYreL1EICIRwHfAU6r6YbF1nwJPq+oi9/23wJ9UtdRLfisRGGMK5eblsid7D6mZqWw7tI0Xl77Ist3LaNuoLZMGTGJkl5EEBgT6OsxqwWdPFotIMPABML14EnDtAjwHHYxzlxljzBmFBIUQ3yCefi37MbrraJbcsYSPR35MWHAYN8+6mcRXE5n580wKtIwBuo1XG4sFmAYcVNX7StlmGPD/ONlYPEVVe5W1XysRGGPOpEALmPnzTB6f/zgb0zcSHRZNs8hmRIdFEx0eTUxYDNHh0TSPbF7Uo2ptv0PJV3cN9QMWAmuBwnT8MNASQFVfdZPFi8BQnNtHx5ZVLQSWCIwx5ZdfkM+7695lfsp80nLS2H9kf9FrZm7mKdvGRsbSOaYznaM70yWmC/1b9adNwza15rmGanHXUGWxRGCMqQy5ebmkZqbyc9rPrE9b70z717MhfQPH8o4BTlfbgxMGF01x9eJO209+QT7Zx7MJrxNerRuoLREYY0w55Rfks+XgFuanzOfb7d8yb/s8Dhw9AEDbRm2JqBNBVm4WWcezyMrN4mjeUQAahDbg8raXc2X7KxnadigN6zb05WGcxhKBMcacowItYO2+tczdPpeFvy4kryCPyJBIIuu4U0gk4cHhrEtbx2ebPyMtJ41ACaRfy35c2f5KrulwTbUY/9kSgTHGVIH8gnyW7lrKp5s/Zfbm2azdvxaA/q36c1vSbdzQ6QbC64T7JDZLBMYY4wMph1N4d927/Gvlv9hycAuRdSIZ2WUkt3e7nV6xvaq0IdoSgTHG+JCqsujXRby+8nXe//l9ck7k0LZRWzpHd6Z1w9ZFU0KDBOIbxHtlDGhLBMYYU01k5mYyY90MPt3yKdsObWPboW3knMgpWi8I7Rq3O2Xchm5Nu1V47AZLBMYYU02pKvuP7GfboW1sP7ydLQe2FHXTvf3w9qLtmkY05YELH+D+C+4/p+/xeadzxhhjSiYiNIloQpOIJlzQ4oJT1h0+dpjVe1ezcu9KVu1dRbOIZl6JwRKBMcZUUw1CGzAgfgAD4gd49Xu82umcMcaY6s8SgTHG+DlLBMYY4+csERhjjJ+zRGCMMX7OEoExxvg5SwTGGOPnLBEYY4yfq3FdTIhIGrDjHD8eBaRXYjg1ib8eux23f7HjLl0rVY0uaUWNSwQVISLLS+tro7bz12O34/YvdtznxqqGjDHGz1kiMMYYP+dviWCqrwPwIX89djtu/2LHfQ78qo3AGGPM6fytRGCMMaYYSwTGGOPn/CYRiMhQEdkkIltFZKKv4/EWEfmXiOwXkXUeyxqJyNcissV9bejLGL1BRFqIyDwR+VlE1ovIve7yWn3sIhIqIktFZLV73E+4yxNEZIn79z5DROr4OlZvEJFAEVkpIp+672v9cYtIioisFZFVIrLcXVahv3O/SAQiEgi8BFwOdAJuEpFOvo3Ka94EhhZbNhH4VlXbAd+672ubPOAPqtoJ6APc7f4b1/ZjzwUGq2pXIAkYKiJ9gL8Bz6lqW+AQcLsPY/Sme4ENHu/95bgHqWqSx7MDFfo794tEAPQCtqrqNlU9DrwLXO3jmLxCVRcAB4stvhqY5s5PA66p0qCqgKruUdWf3PksnJNDLLX82NWR7b4NdicFBgMz3eW17rgBRCQOGAb8030v+MFxl6JCf+f+kghigZ0e71PdZf6iiarucef3Ak18GYy3iUg80A1Ygh8cu1s9sgrYD3wN/AIcVtU8d5Pa+vc+GXgQKHDfN8Y/jluBr0RkhYiMc5dV6O/cBq/3M6qqIlJr7xkWkQjgA+A+Vc10LhIdtfXYVTUfSBKRBsAsoIOPQ/I6ERkO7FfVFSIy0NfxVLF+qrpLRGKAr0Vko+fKc/k795cSwS6ghcf7OHeZv9gnIs0A3Nf9Po7HK0QkGCcJTFfVD93FfnHsAKp6GJgHXAA0EJHCC73a+PfeF7hKRFJwqnoHA89T+48bVd3lvu7HSfy9qODfub8kgmVAO/eOgjrASOATH8dUlT4BbnXnbwU+9mEsXuHWD78ObFDVZz1W1epjF5FotySAiNQFLsFpH5kH3OBuVuuOW1UfUtU4VY3H+f88V1VHUcuPW0TCRSSycB64FFhHBf/O/ebJYhG5AqdOMRD4l6o+5eOQvEJE/gMMxOmWdh/wOPAR8B7QEqcL79+oavEG5RpNRPoBC4G1nKwzfhinnaDWHruIJOI0DgbiXNi9p6pPikhrnCvlRsBK4GZVzfVdpN7jVg39UVWH1/bjdo9vlvs2CHhHVZ8SkcZU4O/cbxKBMcaYkvlL1ZAxxphSWCIwxhg/Z4nAGGP8nCUCY4zxc5YIjDHGz1kiMMYlIvluj46FU6V1UCci8Z49whpTnVgXE8acdFRVk3wdhDFVzUoExpyB2//7390+4JeKSFt3ebyIzBWRNSLyrYi0dJc3EZFZ7hgBq0XkQndXgSLymjtuwFfuk8CIyD3uOAprRORdHx2m8WOWCIw5qW6xqqERHusyVPV84EWcJ9QBXgCmqWoiMB2Y4i6fAnznjhHQHVjvLm8HvKSqnYHDwPXu8olAN3c/4711cMaUxp4sNsYlItmqGlHC8hScwV+2uR3b7VXVxiKSDjRT1RPu8j2qGiUiaUCcZ9cGbtfYX7sDhyAifwKCVfUvIvIlkI3TFchHHuMLGFMlrERgTPloKfNnw7PPm3xOttENwxlBrzuwzKP3TGOqhCUCY8pnhMfrD+78YpyeLwFG4XR6B85QgROgaNCY+qXtVEQCgBaqOg/4E1AfOK1UYow32ZWHMSfVdUf6KvSlqhbeQtpQRNbgXNXf5C77HfCGiDwApAFj3eX3AlNF5HacK/8JwB5KFgi87SYLAaa44woYU2WsjcCYM3DbCJJVNd3XsRjjDVY1ZIwxfs5KBMYY4+esRGCMMX7OEoExxvg5SwTGGOPnLBEYY4yfs0RgjDF+7v8D/6P7y42i53cAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["plt.plot(epochs, test_phoneme_accs, 'b', label='Validation accuracy')\n","plt.title('Validation accuracy')\n","plt.xlabel('Epochs')\n","plt.ylabel('Accuracy')\n","plt.legend()\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":295},"id":"AGqd2g46EMQF","executionInfo":{"status":"ok","timestamp":1639665555402,"user_tz":300,"elapsed":240,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"d330d12c-70dc-4a65-f574-430ecc27ec2c"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZQU5dn+8e/NAA6LAdlcGHRQUMQFxQEEFzaNqAQEN3BF86qoaNQYQxLDzxD1GOVVY0QSjHE384oLQSMSbHCNyCKgghsqCQgiILtsw9y/P6oGeoaemQanqV6uzzl9pmvrvrvFurqep+opc3dEREQqqhV1ASIikp4UECIikpACQkREElJAiIhIQgoIERFJSAEhIiIJKSAkY5mZm1mb8Pmfzey3yay7G+9zoZn9a3frFMlUpusgJCpm9iow3d1HVJjfH/gLUODuJVVs70Bbd1+QxHslta6ZFQJfAXWqem+RXKAjCInS48BFZmYV5l8MPK0ddGqZWe2oa5D0poCQKI0HmgInlc0ws32AvsATZtbZzN41s9VmttTMHjSzuoleyMweM7Pb46Z/EW6zxMwur7DumWY228zWmtkiM7stbvGb4d/VZrbezLqa2RAzeztu+25mNsPM1oR/u8Ute93Mfm9m75jZOjP7l5k1q6TmfczsZTNbbmarwucFccubmNmj4WdYZWbj45b1N7M54Wf4wsz6hPMXmtkpcevdZmZPhc8Lw6a2n5rZf4Ep4fxxZvZN+HneNLMj4ravZ2b/a2b/CZe/Hc77p5ldV+HzfGBmAxJ9VslMCgiJjLtvBJ4FLombfR7wibvPBbYBNwLNgK5Ab+Ca6l433FneDJwKtAVOqbDKhvA9GwNnAleb2VnhspPDv43dvaG7v1vhtZsA/wQeIAi3e4F/mlnTuNUuAC4DWgB1w1oSqQU8ChwEHAhsBB6MW/4kUB84Inyt+8IaOgNPAL8IP8PJwMLKvo8EugOHA6eF0xMJvqcWwPvA03HrjgKOA7oBTYBbgFLCo7+ylcysA9CS4LuRbOHueugR2QM4EVgN5IfT7wA3VrLuDcCLcdMOtAmfPwbcHj7/G3BX3HqHxq+b4HXvB+4LnxeG69aOWz4EeDt8fjFBv0n89u8CQ8LnrwO3xi27Bng1ye/iGGBV+Hx/gh3xPgnW+0tZvQmWLQROiZu+DXiqwmc7uIoaGofrNCIIsI1AhwTr5QOrCPp1IAiSh6L+96RHzT50BCGRcve3gRXAWWZ2CNAZeAbAzA4Nm12+MbO1wJ0ERxPVOQBYFDf9n/iFZtbFzKaGTTtrgKFJvm7Za/+nwrz/EPx6LvNN3PPvgYaJXsjM6pvZX8Lmm7UEzVuNzSwPaAV85+6rEmzaCvgiyXoT2f7dmFmemd0VNlOtZceRSLPwkZ/ovdx9E/B/BH1ItYDBBEc8kkUUEJIOniBo8rkImOTuy8L5Y4BPCH6l/gj4NVCxQzuRpQQ70TIHVlj+DDABaOXujYA/x71udaf1LSFoEop3IPB1EnVV9HPgMKBL+PnKmreMYCfexMwaJ9huEXBIJa+5gaBZqsx+CdaJ/4wXAP0JmuEaERxllNWwAthUxXs9DlxI0PT3vVdojpPMp4CQdPAEwQ7qCoKdTpm9gbXAejNrB1yd5Os9Cwwxs/ZmVh/4fxWW703w63xT2J5/Qdyy5QRNOwdX8tqvAIea2QVmVtvMzgfaAy8nWVvFOjYSdIg3ia/T3ZcS9A08FHZm1zGzsgB5BLjMzHqbWS0zaxl+PwBzgEHh+kXAOUnUsBlYSRAsd8bVUErQXHevmR0QHm10NbO9wuXvEnxX/4uOHrKSAkIi5+4LgX8DDQh+2Ze5mWDnvQ54mKBJI5nXm0jQrzAFWBD+jXcNMNLM1gEjCAKlbNvvgTuAd8Kzp46v8NorCc6y+jnBTvUWoK+7r0imtgruB+oR/FKfBrxaYfnFwFaCo6hvCfpgcPfpBJ3g9wFrgDfYcVTzW4Jf/KuA3xE211XhCYImsq+B+WEd8W4GPgRmAN8Bf6D8fuMJ4CjgqWreRzKQLpQTkd1mZpcAV7r7iVHXIjVPRxAislvC5rtrgLFR1yKpoYAQkV1mZqcR9Ncso/pmLMlQamISEZGEdAQhIiIJZc1gXc2aNfPCwsKoyxARySizZs1a4e7NEy3LmoAoLCxk5syZUZchIpJRzKziyADbqYlJREQSUkCIiEhCCggREUkoa/ogEtm6dSuLFy9m06ZNUZciaSI/P5+CggLq1KkTdSkiaS+rA2Lx4sXsvffeFBYWYjvd1VJyjbuzcuVKFi9eTOvWraMuRyTtZXUT06ZNm2jatKnCQQAwM5o2baojSpEkZXVAAAoHKUf/HkSSl9VNTCIi2cQdVq+GpUuDx5Ilwd9GjeCqq2r+/bL+CCJKPXv2ZNKkSeXm3X///Vx9deX3venRo8f2C/7OOOMMVq9evdM6t912G6NGjaryvcePH8/8+fO3T48YMYLXXnttV8oXkT3EHVauhI8+gsmT4fHH4a674Prr4dxz4cQT4eCDoX59aNIEjjgCTjkFLrkEfvnLYP1U0BFECg0ePJji4mJOO+207fOKi4u5++67k9r+lVde2e33Hj9+PH379qV9+/YAjBw5crdfKyrbtm0jLy8v6jJEdltpabDjr/iLv+xv/PMtW3be/kc/ggMOgP33h65ddzwv+1v22Hvv1NSvgEihc845h1tvvZUtW7ZQt25dFi5cyJIlSzjppJO4+uqrmTFjBhs3buScc87hd7/73U7blw0f0qxZM+644w4ef/xxWrRoQatWrTjuuOMAePjhhxk7dixbtmyhTZs2PPnkk8yZM4cJEybwxhtvcPvtt/P888/z+9//nr59+3LOOecQi8W4+eabKSkpoVOnTowZM4a99tqLwsJCLr30Ul566SW2bt3KuHHjaNeuXbmaFi5cyMUXX8yGDRsAePDBB+nWrRsAf/jDH3jqqaeoVasWp59+OnfddRcLFixg6NChLF++nLy8PMaNG8eiRYsYNWoUL78c3KVz2LBhFBUVMWTIEAoLCzn//POZPHkyt9xyC+vWrdvp89WvX59ly5YxdOhQvvzySwDGjBnDq6++SpMmTbjhhhsA+M1vfkOLFi342c9+lpr/wJKzSkthxYqdd/gVd/zffANbt+68fePGO3byJ51U+Y6/QYM9/9ni5UxA3HADzJlTs695zDFw//2VL2/SpAmdO3dm4sSJ9O/fn+LiYs477zzMjDvuuIMmTZqwbds2evfuzQcffMDRRx+d8HVmzZpFcXExc+bMoaSkhI4dO24PiIEDB3LFFVcAcOutt/LII49w3XXX0a9fv+2BEG/Tpk0MGTKEWCzGoYceyiWXXMKYMWO271SbNWvG+++/z0MPPcSoUaP461//Wm77Fi1aMHnyZPLz8/n8888ZPHgwM2fOZOLEifzjH//gvffeo379+nz33XcAXHjhhQwfPpwBAwawadMmSktLWbRoUZXfa9OmTXn//fcBWLlyZcLPd/3119O9e3defPFFtm3bxvr16znggAMYOHAgN9xwA6WlpRQXFzN9+vQq30ukok2b4Ouvq34sWZJ4x9+kyY6d/GGHVb7jr1dvz3+u3ZEzARGVsmamsoB45JFHAHj22WcZO3YsJSUlLF26lPnz51caEG+99RYDBgygfv36APTr12/7so8++ohbb72V1atXs379+nLNWYl8+umntG7dmkMPPRSASy+9lNGjR28PiIEDBwJw3HHH8cILL+y0/datWxk2bBhz5swhLy+Pzz77DIDXXnuNyy67bHuNTZo0Yd26dXz99dcMGDAACC5SS8b5559f7eebMmUKTzzxBAB5eXk0atSIRo0a0bRpU2bPns2yZcs49thjadq0aVLvKdmvrJ2/up3/ypU7b9ugAbRsGTxOOmnH8wMO2LHz328/SPKfeMbImYCo6pd+KvXv358bb7yR999/n++//57jjjuOr776ilGjRjFjxgz22WcfhgwZstvn5g8ZMoTx48fToUMHHnvsMV5//fUfVO9ee+0FBDvdkpKSnZbfd9997LvvvsydO5fS0tKkd/rxateuTWlp6fbpip+9Qdxx9a5+vv/5n//hscce45tvvuHyyy/f5dokM23eHPyqr+5X/+bN5bczgxYtgp39QQdBt27B84KCHSHQsmXQF5CLZ0jnTEBEpWHDhvTs2ZPLL7+cwYMHA7B27VoaNGhAo0aNWLZsGRMnTqRHjx6VvsbJJ5/MkCFD+NWvfkVJSQkvvfQSV4XntK1bt47999+frVu38vTTT9OyZUsA9t57b9atW7fTax122GEsXLiQBQsWbG/T7969e9KfZ82aNRQUFFCrVi0ef/xxtm3bBsCpp57KyJEjufDCC7c3MTVp0oSCggLGjx/PWWedxebNm9m2bRsHHXQQ8+fPZ/PmzWzcuJFYLMaJJya+531ln693797bm8bKmpgaNWrEgAEDGDFiBFu3buWZZ3QnzExWWgqrVsG338KyZcHfssc335Tf+a9YsfP29ert2MF37Vp+h1/22H9/0KgrlVNA7AGDBw9mwIABFBcXA9ChQweOPfZY2rVrR6tWrTjhhBOq3L5jx46cf/75dOjQgRYtWtCpU6fty37/+9/TpUsXmjdvTpcuXbaHwqBBg7jiiit44IEHeO6557avn5+fz6OPPsq55567vZN66NChSX+Wa665hrPPPpsnnniCPn36bP+136dPH+bMmUNRURF169bljDPO4M477+TJJ5/kqquuYsSIEdSpU4dx48Zx8MEHc95553HkkUfSunVrjj322Erfr7LP98c//pErr7ySRx55hLy8PMaMGUPXrl2pW7cuPXv2pHHjxjoDKg1t3Fh+R59o5182vXw5hL8/yqlVC5o1C3bwrVrB8ccn3vk3bpybv/prUtbck7qoqMgr3jDo448/5vDDD4+oIolCaWkpHTt2ZNy4cbRt2zbhOsn+u9i8GSZODJom9toraF+u+Khsfn4+1M6Bn1+lpfDdd9Xv7MseCQ5qgaCNv0UL2Hff4G/ZI9F0kyag7K85ZjbL3YsSLcuBf8KSK+bPn0/fvn0ZMGBApeFQndJSeOcdePJJGDcuuGp1d9WqtWuhUt18s6C++If7zvN2Z51k1yspCTpxy3b4Vf3Kb958x469c+edd/ZlO/zmzaM/nVMSS2lAmFkf4I9AHvBXd7+rwvKhwLXANmA9cKW7zw+XHQ38BfgRUAp0cneNsiaVat++/fbrInbVJ58EofD00/Cf/wRXrA4cCBddFJzOvHlzcPrjpk3lnyd6VLW84rJ16ypfnug0yt1Vq9aOh1n56eoe8evn5UHTplBYCF26VP5rX7/ys0PKAsLM8oDRwKnAYmCGmU0oC4DQM+7+53D9fsC9QB8zqw08BVzs7nPNrCmwW/+7uLsGaJPt4ptUly2Dv/8dnnoKZs0KdoCnngq33w5nnQUNG0ZYKMEv8/jAgKp33lXt4EV2RyqPIDoDC9z9SwAzKwb6A9sDwt3Xxq3fACj7v/fHwAfuPjdcL8GZydXLz89n5cqVGvJbgCAcli9fycqV+Zx+ejDmzbZt0LEj3HsvDB4cnMueLvLygiOZ8NISkT0ulQHREoi/ZHYx0KXiSmZ2LXATUBfoFc4+FHAzmwQ0B4rdfacBjMzsSuBKgAMPPHCnAgoKCli8eDHLly//YZ9EMpp78At8/Xr46KN8br21gL33hltuCZqQwuGqRKSCyDup3X00MNrMLgBuBS4lqOtEoBPwPRALe9pjFbYdC4yF4Cymiq9dp04d3TksR7nD3LlB89Ezz+wYEvncc+HFF4OrYWtpLGORKqUyIL4GWsVNF4TzKlMMjAmfLwbedPcVAGb2CtARiFWyrQgAixYFgfDkkzBvXnAR1BlnBEcKfftm31AIIqmUyoCYAbQ1s9YEwTAIuCB+BTNr6+6fh5NnAmXPJwG3mFl9YAvQHbgvhbVKBluzBp5/PgiFN94Ijh66doWHHoLzzgvOuhGRXZeygHD3EjMbRrCzzwP+5u7zzGwkMNPdJwDDzOwUgjOUVhE0L+Huq8zsXoKQceAVd/9nqmqVzLNlC0yaFDQhTZgQ9DG0aQO33QYXXgiHHBJ1hSKZL6uvpJbs4g7vvReEQnFxcMFWs2YwaFDQhNS5s07pFNlVupJaMtqCBcEFbE89FTzPz4d+/eDii+G00zTYmkiqKCAkLS1eHDQdPfUUvPtucGTQowf8+tfBFc6NGkVdoUj2U0BI5DZtCq5knjYtCINp04IhnCG4Oftdd8EFFwQjd4rInqOAkD3KPRjrqCwI3n03uBVs2bhDhYVw8snBEM49esBRR6lfQSQqCghJqQ0bgqOD+EBYtixYVq8edOoEN90UBMLxx6fXUBciuU4BITXGHb74onxT0dy5O4aDbtMGfvzjIAi6doUjj1QHs0g6U0DIblu3DmbM2BEG06btuPVjw4bBcNDDh+84OmjWLNp6RWTXKCAkKaWl8Nln5Y8OPvoomA/Qrh385Cc7jg7at9f9AEQynQJCElqzJrgorSwQ3nsvuIE8BKeYdukCAwYEgdClC+yzT7T1ikjNU0BIOZMnw403wvz5QZ+CWXCq6Tnn7Dg6OOwwjYQqkgsUELLd3LnBRWgtW8LIkUEgdO4MP/pR1JWJSBQUEALAkiXBcNiNGkEsFoSEiOQ2BYSwYUMwttGqVfD22woHEQkoIHLctm3BSKizZ8M//gHHHBN1RSKSLhQQOW74cBg/Hu6/P2hiEhEpo3NRctjYsTBqFFx7LVx/fdTViEi6UUDkqMmT4Zpr4PTTg6MHDYgnIhUpIHLQvHnBdQ3t2wd3ZquthkYRSUABkWOWLYMzz4T69eHll3WNg4hUTr8dc8jGjdC/P3z7Lbz5Jhx4YNQViUg6U0DkiNJSuPRSmD4dnn8eihLeolxEZAcFRI747W9h3Di4++5gkD0RkeqoDyIHPPoo3HknXHEF3Hxz1NWISKZQQGS5qVPhyivhlFNg9GidzioiyUtpQJhZHzP71MwWmNnwBMuHmtmHZjbHzN42s/YVlh9oZuvNTL97d8Onn8LZZ8OhhwbNS7q9p4jsipQFhJnlAaOB04H2wOCKAQA84+5HufsxwN3AvRWW3wtMTFWN2WzFiuB01tq1g9NZGzeOuiIRyTSp7KTuDCxw9y8BzKwY6A/ML1vB3dfGrd8A8LIJMzsL+ArYkMIas9LmzUFH9OLFQRNT69ZRVyQimSiVTUwtgUVx04vDeeWY2bVm9gXBEcT14byGwC+B31X1BmZ2pZnNNLOZy5cvr7HCM5k7XH55MGz3448Hd4ATEdkdkXdSu/todz+EIBBuDWffBtzn7uur2Xasuxe5e1Hz5s1TXGlmGDkSnnkGbr8dzj8/6mpEJJOlsonpa6BV3HRBOK8yxcCY8HkX4BwzuxtoDJSa2SZ3fzAllWaJp5+G224LLoj79a+jrkZEMl0qA2IG0NbMWhMEwyDggvgVzKytu38eTp4JfA7g7ifFrXMbsF7hULW33w6alrp3D4bx1umsIvJDpSwg3L3EzIYBk4A84G/uPs/MRgIz3X0CMMzMTgG2AquAS1NVTzZbsADOOgsKC+GFF6Bu3agrEpFsYO5e/VoZoKioyGfOnBl1GXvcqlVw/PHBaa3vvQdt2kRdkYhkEjOb5e4JR2fTWEwZbMsWGDgQFi6E115TOIhIzVJAZCh3uOoqeP11ePJJOOmkajcREdklkZ/mKrvnrrvgscdgxAi46KKoqxGRbKSAyEDPPhucxjp4cHBaq4hIKiggMsy0aXDJJXDCCfC3v+l0VhFJHQVEBlm4MLhlaMuW8OKLkJ8fdUUiks0UEBlizZpgdNYtW+Cf/wSNLCIiqaazmDLA1q1w7rnw2WcwaRK0axd1RSKSCxQQac4dhg2DyZODPodevaKuSERyhZqY0ty99wZjKw0fDpddFnU1IpJLFBBpbPx4+MUv4Jxz4I47oq5GRHKNAiJNzZoFF1wAnTrBE09ALf2XEpE9TLudNLRoEfzkJ9CiBUyYAPXqRV2RiOQidVKnmXXroG9f2LAB/vUv2HffqCsSkVylgEgjJSUwaBDMmxdc63DkkVFXJCK5TAGRRm66CV55BcaMgdNOi7oaEcl16oNIE2+8AX/6E9x4IwwdGnU1IiIKiLTx6qtQuzaMHBl1JSIiAQVEmojFgluHNmwYdSUiIgEFRBpYvTq47qF376grERHZQQGRBl5/HUpLFRAikl4UEGkgFoP69aFLl6grERHZQQGRBmIxOPlkqFs36kpERHZQQERsyRL4+GM1L4lI+klpQJhZHzP71MwWmNnwBMuHmtmHZjbHzN42s/bh/FPNbFa4bJaZZe1dEKZMCf4qIEQk3aQsIMwsDxgNnA60BwaXBUCcZ9z9KHc/BrgbuDecvwL4ibsfBVwKPJmqOqMWi0GTJtChQ9SViIiUl8ojiM7AAnf/0t23AMVA//gV3H1t3GQDwMP5s919STh/HlDPzPZKYa2RcA8ComdPDectIumn2t2Smf3EzHZn99USWBQ3vTicV/H1rzWzLwiOIK5P8DpnA++7++YE215pZjPNbOby5ct3o8RoLVgQDO2t5iURSUfJ7PjPBz43s7vNrF1NF+Duo939EOCXwK3xy8zsCOAPwFWVbDvW3Yvcvah58+Y1XVrKxWLBXwWEiKSjagPC3S8CjgW+AB4zs3fDX+57V7Pp10CruOmCcF5lioGzyibMrAB4EbjE3b+ors5MFItBQQG0bRt1JSIiO0uq6SjsK3iOYCe+PzAAeN/MrqtisxlAWzNrbWZ1gUHAhPgVzCx+13gm8Hk4vzHwT2C4u7+T5GfJKKWlMHVqcPRgFnU1IiI7S6YPop+ZvQi8DtQBOrv76UAH4OeVbefuJcAwYBLwMfCsu88zs5Fm1i9cbZiZzTOzOcBNBGcsEW7XBhgRngI7x8xa7N5HTE9z58LKlWpeEpH0lcwNg84G7nP3N+Nnuvv3ZvbTqjZ091eAVyrMGxH3/GeVbHc7cHsStWUs9T+ISLpLJiBuA5aWTZhZPWBfd1/o7rFUFZbtYjFo1w4OOCDqSkREEkumD2IcUBo3vS2cJ7tpyxZ46y0dPYhIeksmIGqHF7oBED7XsHI/wPTpsGGDAkJE0lsyAbE8rlMZM+tPMBSG7KZYLLhyukePqCsREalcMn0QQ4GnzexBwAiujr4kpVVluVgMOnaEffaJuhIRkcpVGxDhRWrHm1nDcHp9yqvKYhs2wLRpcNNNUVciIlK1ZI4gMLMzgSOAfAuv6nL3kSmsK2u99RZs3ar+BxFJf8lcKPdngvGYriNoYjoXOCjFdWWtWCy4c9wJJ0RdiYhI1ZLppO7m7pcAq9z9d0BX4NDUlpW9YjHo1i24B7WISDpLJiA2hX+/N7MDgK0E4zHJLlq5EubMUfOSiGSGZPogXgoHz7sHeJ/gpj4Pp7SqLDV1anCToF5ZewNVEckmVQZEeKOgmLuvBp43s5eBfHdfs0eqyzKxGDRsCJ06RV2JiEj1qmxicvdSgvtKl01vVjjsvlgMuneHOnWirkREpHrJ9EHEzOxsM9214IdYtAg+/1z9DyKSOZIJiKsIBufbbGZrzWydma1NcV1ZR8N7i0imSeZK6upuLSpJiMWgeXM48sioKxERSU61AWFmJyeaX/EGQlI59yAgevUKBukTEckEyZzm+ou45/lAZ2AWoJM1k/TJJ7B0qZqXRCSzJNPE9JP4aTNrBdyfsoqykPofRCQT7U6Dx2Lg8JouJJtNmQKFhXDwwVFXIiKSvGT6IP5EcPU0BIFyDMEV1ZKEbduCK6jPPjvqSkREdk0yfRAz456XAH9393dSVE/WmT0bVq9W85KIZJ5kAuI5YJO7bwMwszwzq+/u36e2tOxQ1v+g8ZdEJNMkdSU1UC9uuh7wWjIvbmZ9zOxTM1tgZsMTLB9qZh+a2Rwze9vM2sct+1W43admdloy75eOYrHg2od99426EhGRXZNMQOTH32Y0fF7t3QzMLI9gHKfTgfbA4PgACD3j7ke5+zHA3cC94bbtgUEEd7HrAzwUvl5G2bwZ3n5bzUsikpmSCYgNZtaxbMLMjgM2JrFdZ2CBu3/p7luAYqB//AruHj9kRwN2dIb3B4rDwQG/AhaEr5dR3n0XNm5UQIhIZkqmD+IGYJyZLSG45eh+BLcgrU5LYFHc9GKgS8WVzOxa4CagLjsuvmsJTKuwbcsk3jOtxGKQlxeM4CoikmmSuVBuhpm1Aw4LZ33q7ltrqgB3Hw2MNrMLgFuBS5Pd1syuBK4EOPDAA2uqpBoTiwX3fvjRj6KuRERk11XbxBT+wm/g7h+5+0dAQzO7JonX/hpoFTddEM6rTDFw1q5s6+5j3b3I3YuaN2+eREl7ztq1MH26zl4SkcyVTB/EFeEd5QBw91XAFUlsNwNoa2atzawuQafzhPgVzKxt3OSZwOfh8wnAIDPby8xaA22B6Um8Z9p4883gIjn1P4hIpkqmDyLPzMzdHbafnVS3uo3cvcTMhgGTgDzgb+4+z8xGAjPdfQIwzMxOAbYCqwibl8L1ngXmE1ycd23ZdRiZIhaD/Hzo1i3qSkREdo+F+/3KVzC7BzgI+Es46yrgv+5+c4pr2yVFRUU+c+bM6lfcQ44+Glq0gNeSumJERCQaZjbL3YsSLUumiemXwBRgaPj4kPIXzkkF334LH36o5iURyWzVBoS7lwLvAQsJrkXoBXyc2rIy25QpwV8FhIhkskr7IMzsUGBw+FgB/B+Au/fcM6VlrlgMGjWC446LuhIRkd1XVSf1J8BbQF93XwBgZjfukaoyXCwGPXoEF8mJiGSqqpqYBgJLgalm9rCZ9Sa4klqq8NVXwUPNSyKS6SoNCHcf7+6DgHbAVIIhN1qY2Rgz+/GeKjDTqP9BRLJFMp3UG9z9mfDe1AXAbIIzmySBWAz23x8O101ZRSTD7dI9qd19VTi8hX4fJ+AeHEH06gWmxjgRyXC7FBBStXnzYNkyNS+JSHZQQNSgstuLKiBEJBsoIGpQLAZt2kAajjwuIrLLFBA1pKQE3nhDRw8ikj0UEDVk5szgHhAKCBHJFgqIGlLW/9BTA5GISJZQQNSQWAw6dIBmzUhsEcAAAAt/SURBVKKuRESkZiggasDGjfDvf6t5SUSyiwKiBrzzDmzerIAQkeyigKgBsRjUrg0nnxx1JSIiNUcBUQNiMejSBRo2jLoSEZGao4D4gVavhlmz1LwkItlHAfEDvf46lJYqIEQk+yggfqBYDOrXh+OPj7oSEZGapYD4gaZMgZNOgrp1o65ERKRmKSB+gKVLYf58NS+JSHZKaUCYWR8z+9TMFpjZ8ATLbzKz+Wb2gZnFzOyguGV3m9k8M/vYzB4wS79b8Oj2oiKSzVIWEGaWB4wGTgfaA4PNrH2F1WYDRe5+NPAccHe4bTfgBOBo4EigE9A9VbXurlgMmjSBY46JuhIRkZqXyiOIzsACd//S3bcAxUD/+BXcfaq7fx9OTiO45zWAA/lAXWAvoA6wLIW17jL3ICB69oRaaqgTkSyUyl1bS2BR3PTicF5lfgpMBHD3d4GpwNLwMcndP05Rnbvliy/gv/9V85KIZK+0+O1rZhcBRcA94XQb4HCCI4qWQC8zOynBdlea2Uwzm7l8+fI9WbJuLyoiWS+VAfE10CpuuiCcV46ZnQL8Bujn7pvD2QOAae6+3t3XExxZdK24rbuPdfcidy9q3rx5jX+AqsRiUFAAbdvu0bcVEdljUhkQM4C2ZtbazOoCg4AJ8SuY2bHAXwjC4du4Rf8FuptZbTOrQ9BBnTZNTKWlwRlMvXtD+p1bJSJSM1IWEO5eAgwDJhHs3J9193lmNtLM+oWr3QM0BMaZ2RwzKwuQ54AvgA+BucBcd38pVbXuqg8+gJUr1bwkItmtdipf3N1fAV6pMG9E3PNTKtluG3BVKmv7Icr6H3r1irYOEZFUSotO6kwTi8Fhh0HLqs7JEhHJcAqIXbRlC7z5ppqXRCT7KSB20fTpsGGDAkJEsp8CYhfFYsGZSz16RF2JiEhqKSB2USwGHTsGYzCJiGQzBcQu2LABpk1T85KI5AYFxC546y3YulUBISK5QQGxC6ZMCe4cd+KJUVciIpJ6CohdEItB167BPahFRLKdAiJJ330Hs2ereUlEcocCIklTpwY3CVJAiEiuUEAkKRaDhg2hU6eoKxER2TMUEEmKxaB7d6hTJ+pKRET2DAVEEhYvhs8+U/OSiOQWBUQSdHtREclFCogkxGLQvDkceWTUlYiI7DkKiGq4BwHRqxfU0rclIjlEu7xqfPopLFmi5iURyT0KiGro9qIikqsUENWIxeCgg+Dgg6OuRERkz1JAVGHbtuAK6t69g5sEiYjkEgVEFWbPhtWr1f8gIrlJAVEF9T+ISC5TQFQhFoMjjoD99ou6EhGRPS+lAWFmfczsUzNbYGbDEyy/yczmm9kHZhYzs4Pilh1oZv8ys4/DdQpTWWtFmzfD22+reUlEclfKAsLM8oDRwOlAe2CwmbWvsNpsoMjdjwaeA+6OW/YEcI+7Hw50Br5NVa2JTJsGGzcqIEQkd6XyCKIzsMDdv3T3LUAx0D9+BXef6u7fh5PTgAKAMEhqu/vkcL31cevtEbFYcOV09+578l1FRNJHKgOiJbAobnpxOK8yPwUmhs8PBVab2QtmNtvM7gmPSMoxsyvNbKaZzVy+fHmNFQ5BQHTqBI0a1ejLiohkjLTopDazi4Ai4J5wVm3gJOBmoBNwMDCk4nbuPtbdi9y9qHnz5jVWz7p1MH26mpdEJLelMiC+BlrFTReE88oxs1OA3wD93H1zOHsxMCdsnioBxgMdU1hrOW++CSUlCggRyW2pDIgZQFsza21mdYFBwIT4FczsWOAvBOHwbYVtG5tZ2WFBL2B+CmstJxaD/Hzo1m1PvaOISPpJWUCEv/yHAZOAj4Fn3X2emY00s37havcADYFxZjbHzCaE224jaF6KmdmHgAEPp6rWimIxOOGEICRERHJV7VS+uLu/ArxSYd6IuOenVLHtZODo1FWX2LffwgcfwJ137ul3FhFJL2nRSZ1Opk4N/qr/QURynQKiglgsOLX1uOOirkREJFoKiApiMejRA/J2uupCRCS3KCDiLFwIX36p0VtFREABUU7Z8N7qfxARUUCUE4sFQ3u3rzikoIhIDlJAhNxhypSgeUm3FxURUUBsN28eLFum5iURkTIKiJD6H0REylNAhKZMgUMOgYMOqn5dEZFcoIAgGLn19dd19CAiEk8BAcyaBWvXKiBEROIpINjR/9CzZ7R1iIikEwUEQUB06AA1eFM6EZGMl/MBsXEjvPOOmpdERCrK+YBYswYGDoS+faOuREQkvaT0hkGZYL/94Jlnoq5CRCT95PwRhIiIJKaAEBGRhBQQIiKSkAJCREQSUkCIiEhCCggREUlIASEiIgkpIEREJCFz96hrqBFmthz4zw94iWbAihoqJ9PpuyhP30d5+j52yIbv4iB3TzgSXdYExA9lZjPdvSjqOtKBvovy9H2Up+9jh2z/LtTEJCIiCSkgREQkIQXEDmOjLiCN6LsoT99Hefo+dsjq70J9ECIikpCOIEREJCEFhIiIJJTzAWFmfczsUzNbYGbDo64nSmbWysymmtl8M5tnZj+LuqaomVmemc02s5ejriVqZtbYzJ4zs0/M7GMz6xp1TVEysxvD/08+MrO/m1l+1DXVtJwOCDPLA0YDpwPtgcFm1j7aqiJVAvzc3dsDxwPX5vj3AfAz4OOoi0gTfwRedfd2QAdy+Hsxs5bA9UCRux8J5AGDoq2q5uV0QACdgQXu/qW7bwGKgf4R1xQZd1/q7u+Hz9cR7ABaRltVdMysADgT+GvUtUTNzBoBJwOPALj7FndfHW1VkasN1DOz2kB9YEnE9dS4XA+IlsCiuOnF5PAOMZ6ZFQLHAu9FW0mk7gduAUqjLiQNtAaWA4+GTW5/NbMGURcVFXf/GhgF/BdYCqxx939FW1XNy/WAkATMrCHwPHCDu6+Nup4omFlf4Ft3nxV1LWmiNtARGOPuxwIbgJztszOzfQhaG1oDBwANzOyiaKuqebkeEF8DreKmC8J5OcvM6hCEw9Pu/kLU9UToBKCfmS0kaHrsZWZPRVtSpBYDi9297IjyOYLAyFWnAF+5+3J33wq8AHSLuKYal+sBMQNoa2atzawuQSfThIhrioyZGUEb88fufm/U9UTJ3X/l7gXuXkjw72KKu2fdL8Rkufs3wCIzOyyc1RuYH2FJUfsvcLyZ1Q//v+lNFnba1466gCi5e4mZDQMmEZyF8Dd3nxdxWVE6AbgY+NDM5oTzfu3ur0RYk6SP64Cnwx9TXwKXRVxPZNz9PTN7Dnif4Oy/2WThsBsaakNERBLK9SYmERGphAJCREQSUkCIiEhCCggREUlIASEiIgkpIESqYWbbzGxO3KPGriA2s0Iz+6imXk+kJuX0dRAiSdro7sdEXYTInqYjCJHdZGYLzexuM/vQzKabWZtwfqGZTTGzD8wsZmYHhvP3NbMXzWxu+CgbmiHPzB4O7y3wLzOrF65/fXhvjg/MrDiijyk5TAEhUr16FZqYzo9btsbdjwIeJBj9FeBPwOPufjTwNPBAOP8B4A1370AwjlHZVfttgdHufgSwGjg7nD8cODZ8naGp+nAildGV1CLVMLP17t4wwfyFQC93/zIc5PAbd29qZiuA/d19azh/qbs3M7PlQIG7b457jUJgsru3Dad/CdRx99vN7FVgPTAeGO/u61P8UUXK0RGEyA/jlTzfFZvjnm9jR9/gmQR3POwIzAhvTCOyxyggRH6Y8+P+vhs+/zc7bj95IfBW+DwGXA3b73XdqLIXNbNaQCt3nwr8EmgE7HQUI5JK+kUiUr16caPbQnBf5rJTXfcxsw8IjgIGh/OuI7jz2i8I7sJWNurpz4CxZvZTgiOFqwnuRpZIHvBUGCIGPKBbfMqepj4Ikd0U9kEUufuKqGsRSQU1MYmISEI6ghARkYR0BCEiIgkpIEREJCEFhIiIJKSAEBGRhBQQIiKS0P8Hg80BGpJ57UgAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["from collections import defaultdict\n","\n","\n","samples = generate_samples(model, test_loader)\n","for sample in samples[0:10]:\n","  print(sample['target'])\n","  print(sample['predicted'])  \n","  print(sample['target_mapping'])\n","  print(sample['predicted_mapping'])  \n","\n","  print(sample['mapping_ed'])  \n","  print(sample['mapping_acc'])  "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vw3Kp1M047uc","executionInfo":{"status":"ok","timestamp":1639665560200,"user_tz":300,"elapsed":4801,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"d35e0ac8-b0b4-4aed-cbf4-97688963dc57"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["['h#', 'dh', 'ax', 'bcl', 'b', 'ah', 'ng', 'g', 'ax', 'l', 'ow', 'w', 'ax', 'z', 'pcl', 'p', 'l', 'eh', 'z', 'ax', 'n', 'q', 'l', 'iy', 's', 'ih', 'tcl', 'ch', 'uw', 'w', 'ey', 'dx', 'ix', 'dcl', 'n', 'ih', 'axr', 'dh', 'ix', 'sh', 'ao', 'r', 'h#']\n","['h#', 'dh', 'ax', 'l', 'b', 'r', 'n', 'kcl', 'ow', 'l', 'ae', 'l', 'ax', 'n', 'ix', 'p', 'r', 'ax', 'n', 'ix', 'n', 'dcl', 'ae', 'ax', 'n', 'ix', 'n', 't', 'axr', 'ix', 'ax', 'dx', 'ix', 'n', 'd', 'dcl', 'n', 'h#', 'ax', 'n', 'en', 'r', 'ix']\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8]\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 4, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 4, 5, 6, 5, 6, 6, 8, 7, 8, 8, 7, 8, 8, 8, 8, 8, 8]\n","0.6046511627906976\n","0.5813953488372093\n","['h#', 'd', 'ow', 'nx', 'ae', 's', 'kcl', 'k', 'm', 'iy', 'dx', 'ix', 'kcl', 'k', 'eh', 'r', 'ih', 'ix', 'n', 'ax', 'q', 'oy', 'l', 'ih', 'r', 'ae', 'gcl', 'g', 'l', 'ay', 'kcl', 'k', 'dh', 'ae', 'tcl', 'h#']\n","['h#', 'd', 'ow', 'nx', 'ae', 's', 'kcl', 'm', 'm', 'iy', 'dx', 'ix', 'kcl', 'k', 'eh', 'r', 'iy', 'ix', 'n', 'q', 'q', 'oy', 'l', 'iy', 'r', 'ae', 'gcl', 'g', 'l', 'ay', 'kcl', 'dh', 'dh', 'ae', 'tcl', 'h#']\n","[1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10]\n","[1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 10, 10, 10, 10, 10]\n","0.9166666666666666\n","0.9166666666666666\n","['h#', 'q', 'aa', 'r', 'y', 'ux', 'l', 'uh', 'kcl', 'k', 'ix', 'ng', 'f', 'axr', 'ix', 'm', 'pcl', 'p', 'l', 'oy', 'm', 'ix', 'n', 'tcl', 'h#']\n","['h#', 'dh', 'ah', 'r', 'dcl', 'ux', 'z', 'iy', 'dcl', 'k', 'el', 'n', 'kcl', 'r', 's', 'n', 'h#', 'p', 'l', 'iy', 'l', 'h#', 'n', 'h#', 't']\n","[1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]\n","[1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5]\n","0.8\n","0.76\n","['h#', 'sh', 'iy', 'hv', 'eh', 'dcl', 'y', 'axr', 'dcl', 'd', 'aa', 'r', 'kcl', 'k', 's', 'ux', 'tcl', 'en', 'g', 'r', 'iy', 's', 'iy', 'w', 'ao', 'sh', 'epi', 'w', 'ao', 'dx', 'axr', 'q', 'ao', 'l', 'y', 'ih', 'er', 'h#']\n","['h#', 'sh', 'iy', 'hv', 'ae', 'dcl', 'jh', 'axr', 'dcl', 'd', 'aa', 'r', 'kcl', 'k', 's', 'ux', 'tcl', 'en', 'gcl', 'r', 'iy', 's', 'iy', 'w', 'aa', 'sh', 'epi', 'w', 'ao', 'dx', 'axr', 'ao', 'ao', 'l', 'y', 'ih', 'axr', 'h#']\n","[1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 11, 11, 11, 11]\n","[1, 1, 1, 2, 2, 2, 2, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 11, 11, 11, 11]\n","0.9736842105263158\n","0.9736842105263158\n","['h#', 'q', 'ix', 'tcl', 't', 'w', 'ay', 'l', 'ay', 'dx', 'ao', 'n', 'dh', 'ax', 'tcl', 't', 'w', 'eh', 'l', 'f', 'th', 'dcl', 'd', 'ey', 'pau', 'w', 'ih', 'l', 'hv', 'ae', 'v', 'sh', 'ix', 'bcl', 'b', 'l', 'iy', 'h#']\n","['h#', 'dh', 'ah', 'n', 't', 'r', 'ah', 'kcl', 'dcl', 'dcl', 'iy', 'r', 'tcl', 'ax', 'l', 't', 'r', 'ax', 'l', 'dcl', 'iy', 'r', 'd', 'ix', 'tcl', 'q', 'ax', 'n', 'iy', 'ae', 'n', 'axr', 'ix', 'n', 'b', 'ay', 'iy', 'dcl']\n","[1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9]\n","0.7105263157894737\n","0.5789473684210527\n","['h#', 'q', 'iy', 'dx', 'ih', 'ng', 's', 'pcl', 'p', 'ih', 'n', 'ix', 'tcl', 'ch', 'n', 'ay', 'tcl', 'l', 'iy', 'ix', 'n', 'kcl', 'k', 'r', 'iy', 's', 'ix', 's', 'tcl', 't', 'r', 'ih', 'ng', 'th', 'm', 'er', 'ae', 'kcl', 'k', 'y', 'ax-h', 'l', 'ax', 's', 'l', 'iy', 'h#']\n","['h#', 'b', 'ae', 'kcl', 'ih', 'kcl', 'kcl', 'kcl', 'p', 'r', 'n', 'tcl', 'n', 't', 'ix', 'tcl', 's', 't', 'iy', 'z', 'n', 'tcl', 'k', 'r', 'ae', 'z', 'ix', 'n', 'tcl', 't', 'r', 'ae', 'n', 'h#', 'r', 'eh', 's', 'n', 'k', 'el', 'ux', 'kcl', 'iy', 'm', 'h#', 'iy', 'h#']\n","[1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6]\n","[1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 3, 3, 3, 4, 4, 4, 5, 4, 6, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6]\n","0.7021276595744681\n","0.7021276595744681\n","['h#', 'g', 'aa', 'dx', 'ix', 'hv', 'eh', 'kcl', 'k', 'ax', 'v', 'ax', 'bcl', 'b', 'ay', 'aa', 'n', 'dh', 'ih', 's', 'pau', 'd', 'er', 'tcl', 'ch', 'iy', 'pcl', 'p', 'h#']\n","['h#', 'dh', 'r', 'r', 'ix', 'kcl', 'ae', 'dcl', 'k', 'r', 'n', 'ix', 'n', 'b', 'ay', 'tcl', 'r', 'tcl', 'ax', 'n', 'tcl', 'q', 'ih', 'dcl', 't', 'ae', 'q', 'p', 'r']\n","[1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 4, 5, 6, 6, 6, 7, 7, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10]\n","[1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 3, 4, 4, 4, 4, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, 6]\n","0.24137931034482762\n","0.1724137931034483\n","['h#', 'dh', 'ax', 's', 'kcl', 'k', 'ae', 'l', 'ax', 'pcl', 't', 'eh', 'dcl', 'jh', 'ix', 'z', 'pcl', 'p', 'axr', 'tcl', 't', 'ih', 'kcl', 'k', 'y', 'ix', 'dcl', 'l', 'iy', 'ix', 'pcl', 'p', 'iy', 'l', 'ix', 'ng', 'h#']\n","['h#', 'dh', 'ax', 'l', 'ix', 'k', 'aa', 'kcl', 'ey', 'l', 'p', 'ix', 'n', 'd', 'ix', 'n', 'epi', 'p', 'r', 'f', 't', 'ix', 'n', 'k', 'w', 'ux', 'n', 'd', 'iy', 'h#', 'n', 'p', 'r', 'h#', 'iy', 'n', 'h#']\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6]\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]\n","0.5945945945945945\n","0.4864864864864865\n","['h#', 'ax', 'bcl', 'b', 'ih', 'gcl', 'g', 'ow', 'tcl', 'ay', 'dcl', 'd', 'l', 'iy', 'q', 'ae', 'm', 'bcl', 'b', 'el', 'dcl', 'th', 'r', 'ux', 'dh', 'ax', 'f', 'aa', 'r', 'm', 'y', 'aa', 'r', 'dcl', 'd', 'h#']\n","['h#', 'dh', 'l', 'b', 'r', 'l', 'g', 'r', 'l', 't', 'kcl', 'd', 'ix', 'ay', 's', 'ae', 'n', 'pcl', 'b', 'iy', 'tcl', 'd', 'r', 'ae', 'dcl', 'ax', 'l', 'r', 'r', 'iy', 'pcl', 'ux', 'r', 'dcl', 'd', 'h#']\n","[1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8]\n","[1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 8, 7, 8, 8, 8, 8, 8, 8, 8, 8]\n","0.5833333333333333\n","0.3333333333333333\n","['h#', 'dh', 'ih', 's', 'gcl', 'g', 'r', 'ux', 'pcl', 'p', 'ih', 's', 'eh', 'kcl', 'k', 'y', 'ix', 'l', 'er', 'ih', 's', 'tcl', 't', 'q', 'ix', 'n', 'dh', 'eh', 'r', 'pcl', 'p', 'r', 'ow', 'gcl', 'r', 'eh', 'm', 'tcl', 't', 'eh', 'n', 'z', 'tcl', 't', 'ix', 'bcl', 'b', 'iy', 'tcl', 't', 'eh', 'kcl', 'n', 'ax', 'l', 'aa', 'dcl', 'jh', 'ix', 'kcl', 'k', 'el', 'h#']\n","['h#', 'dh', 'ax', 's', 'pcl', 'g', 'r', 'iy', 'z', 'p', 'r', 'n', 'ix', 'n', 'k', 'ix', 'ux', 'n', 'iy', 'dcl', 'n', 'tcl', 't', 'r', 'ix', 'n', 'tcl', 'ax', 'r', 'iy', 'p', 'r', 'iy', 'n', 'g', 'iy', 'n', 'pcl', 't', 'r', 'n', 'tcl', 'ix', 't', 'r', 'n', 'b', 'el', 'ix', 't', 'ix', 'n', 'k', 'tcl', 'l', 'iy', 'r', 'd', 'ix', 'n', 'k', 's', 'h#']\n","[1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11]\n","[1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 7, 8, 7, 8, 8, 8, 8, 9, 8, 8, 9, 9, 9, 9, 9, 9, 10, 9, 10, 11, 11, 9, 11, 10, 11]\n","0.5396825396825398\n","0.36507936507936506\n"]}]},{"cell_type":"code","source":["from collections import defaultdict\n","\n","samples = generate_samples(model, test_loader)\n","len_perf_mapping = defaultdict(lambda: [])\n","\n","\n","for sample in samples:\n","  len_perf_mapping[len(sample['target'])].append(sample['mapping_ed'])\n","\n","lengths = []\n","bleu_scores = []\n","\n","for key, val in len_perf_mapping.items():\n","\n","  lengths.append(key)\n","  bleu_scores.append(sum(val)/len(val))\n"],"metadata":{"id":"1PkZMhxUEa56"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plt.bar(lengths, bleu_scores, color='b')\n","\n","plt.title('Phoneme length vs Mapping Edit Distance')\n","plt.xlabel('Lengths')\n","plt.ylabel('Edit Distance')\n","plt.savefig(WORKSPACE_PATH + '/transformer_length_analysis.pdf')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":295},"id":"X6TEKHEOE4-8","executionInfo":{"status":"ok","timestamp":1639665565714,"user_tz":300,"elapsed":571,"user":{"displayName":"Sahas D","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjFhTlCEcqzOdyxuFJOFIBVlqcxwFeNCZ0oVVfdng=s64","userId":"10479330133775136174"}},"outputId":"c12eade6-6407-4284-8bb9-c002848e9dae"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debgcVZ3/8feHQEAW2XJlIIkkagSiIyAREGYUBTS4hBkWTVxxkIgaBTcGRn+MgjoKiiMjLlEUBmUJqBgwbAo4goK5QUCSGAgBzQ3bBcKuQOD7++OcC5VO9719k1vdt299Xs/TT1ed2s7pru5v1TlVpxQRmJlZda3X7gyYmVl7ORCYmVWcA4GZWcU5EJiZVZwDgZlZxTkQmJlVnANBi0m6WtIH252PoSTpTkn7tWG7EySFpPVbve12kvTPkpa0Ox8A+fN/WR7+rqT/N8Trf7eky4dynbYmB4IS5D/Gv0l6TNK9ks6QtGm789Xp2hVwavKwT/7z+3lN+s45/eqy8xARv42IHcpYdz5Q+Xved/teFzWZryMj4sS8nn0k9QywrTMkPSXp0fy6RdJ/Sdq8sM6fRMSbmsj3GZK+2Ew+bU0OBOV5e0RsCrwamAJ8rs35saHTC7xW0taFtPcDt7YpP0NtVkRsWni9vcRtnRQRmwFdwAeAPYFrJW1S4jathgNBySJiBXAJ8MpC8vaSrs1HQZdLGtM3QdI0SQslPZSPznYqTLtT0qcl3SzpYUnnSdqoMP1tkm7My/5O0qtqlv1MXvZxSadL2kbSJTkfv5K0ZWH+PfM6HpJ0k6R9mimvpPUkHSvpdkkPSJojaas8ra8q5/2S/irpfkmfLSz7AklnSlopabGkY/qOKiWdBbwYuCgfpR5T2Oy7662vJl97SLpH0qhC2r9KujkP7y6pW9Ij+SzulH6K+RRwITA9LzsKeCfwk5ptflPS8rzOBZL+uTDt85IuyN/ho5JukLRzYfqdko6TtCh/Hj/q+65rj7ab2C+OkXS3pLskfVCF6pzByvtQ37r+rWbaGZK+mP/ELwG2K5xVbNffeiPi7xExH5gGbE0KCkg6TNI1eViSviHpvvyZ/knSKyXNBN4NHKPCGUxhP3w0f47/WsjrYZKukfS1/PneIemAwvSt8md+V55+YWFaw99Zx4oIv4b4BdwJ7JeHxwMLgRPz+NXA7cDLgRfk8a/kaS8HHgf2BzYAjgGWAqML6/0DsB2wFbAYODJP2xW4D9gDGEU6Qr0T2LCw7HXANsDYPO8NebmNgCuB/8zzjgUeAN5COljYP493NVHeo/J2xgEbAt8DzsnTJgABfD+XfWfgSWCnPP0rwG+ALfPyNwM99bbTzPrq5PN2YP/C+PnAsXn498B78/CmwJ4N1rEP0APsBVyf094CXAZ8ELi6MO97SH9q6wOfAu4BNsrTPg88DRySv+tPA3cAGxTKegtp/9kKuBb4YjEPNZ9Lo/1iat7uK4CNgR/nz+xlDcp3NfDBBtOmAveSDmo2Ac4urgs4o1EeG6zvuflr0v8XOC8PHwZck4ffDCwAtgAE7ARs22hdwKH5M1mPFKgfL8x/WP78jyD9Xj4M3AUoT/8lcB5pX9wAeH0zv7NOffmMoDwXSnoIuIb05/blwrQfRcStEfE3YA6wS05/J/DLiLgiIp4Gvkb6g9ursOypEXFXRDwIXFRYdibwvYi4PiKeiYgzSX+KexaW/Z+IuDfSWcpvSX9kf4yIvwM/J+3kkP7A5kXEvIh4NiKuALpJf3gDORL4bET0RMSTpD+8Q7R6g+4XIuJvEXETcBPpDxzgHcCXI2JlRPQApzaxvf7WV+scYAaApM1yec7J054GXiZpTEQ8FhHX9bfBiPgdsJWkHYD3kf68auf5cUQ8EBGrIuLrpMBYrNtfEBEX5O/6FFJALn5f34qI5fm7/lJf3htotF+8g7S/LYyIJ0jfx0BOzUe7fa8Ta9Z1S0Q83uS61sZdpIBW62lgM2BH0h/24oi4u9FKIuL8/Jk8GxHnAbcBuxdm+UtEfD8ingHOBLYFtpG0LXAAKZiujIinI+I3eZlmfmcdx4GgPP8SEVtExPYR8ZH8p9/nnsLwE6QjUEhHL3/pmxARzwLLSUfoAy27PfCp4g+YdDRZPCW/tzD8tzrjxXUdWrOufyL9UAayPfDzwnKLgWdIZyIDlWG7XN4+xeH+NFpfrbOBgyRtCBwE3BARfZ/34aQzsj9Lmi/pbU1s9yxgFvAGUiBdTa6uWZyrax4CNgfGFGZ5rnz5u+5h9e+rWP6/1EyrNZSf6cfzvtv36rsSqHZdf6mz7FAYCzxYmxgRVwLfAk4D7pM0W9ILG61E0vsKVTgPkc5kip//c59ZDpKQPrfxwIMRsbLOapv5nXUcB4Lh5S7SjgakOlHSTraiiWWXA1+q+QFvHBHnDLhk/XWdVbOuTSLiK00ue0DNshvls5CB3E2qEuozvmb6OnWVGxGLSH9eBwDvIgWGvmm3RcQM4EXAV4ELNHCD5VnAR0hnT08UJ+T2gGNIR9FbRsQWwMOkKo0+4wvzr0cq+131ppPaR4rTmjXQZzrYddXmqZG1+q6Urq7bj3TGuuZKI06NiN2AyaTA/Zl625O0PanKcBawdf78b2H1z7+R5aSzvS0aTBuq39mw4UAwvMwB3ippX0kbkOqVnwR+18Sy3weOVGoUlaRNJL01V4EM1o+Bt0t6s6RRkjbKDZTjBlwSvgt8Kf8QkdQl6cAmtzsHOE7SlpLGkn7ERfcCL2m2EA2cTWrHeB2pjYCcz/dI6spH5g/l5Gf7W1FE3AG8HqjXQL0ZsIp0hdH6ko4Hao9ed5N0UK42O5r0XRerpD4qaZxSY/tnSXXWgzUH+ICknSRtDKzLdf5zgMMkTc7r+s9+5r0X2FqFS0H7I2lDSbuRGuFXAj+qM89r8v69Aam+/+88/x3V7hubkIJDb172A6x+wUZDubrpEuDbeV/cQNLr8uSh/J0NGw4Ew0hELCHVz/8PcD/wdtJlqE81sWw3qeHrW6Qf0lJSg9ja5GM5cCDwH6Qf0nLSkVcz+8s3gbnA5ZIeJf2x7dHkpk8gVY/cAfwKuID059jnv4DP5VPyTze5zlrnkP68r4yI+wvpU4GFkh7LZZheU51XV0RcExH1jtQvAy4lXVL6F9KfVm21zC9I7UIrgfcCB+X2gj5nA5cDy0gN3YO+Tj4iLiG1tVxF2if6As2TDReCb2n1+wgWFNb136QLC5bm90bb/TPps16Wv69GVSfH5P3kAVI7ywJgr9wGUeuFpD/ilaTP9AHg5DztdGBy3taF+ezv66SLAO4F/pHU4N6s95LaJP5Mahw+OpdryH5nw0lfC7nZsCPpw6Q/5Ne3Oy9DTdLnSVfbvKfB9DtJV+/8aoi3uxOpimTDiFg1lOu2zuUzAhs2JG0raW+lexF2IFWNrdEIa4OjdL/Ehkr3iXwVuMhBwIocCGw4GU267+BRUrXDL4BvtzVHI8OHSNUbt5Ou4Ppwe7Njw42rhszMKs5nBGZmFddx3feOGTMmJkyY0O5smJl1lAULFtwfEV31pnVcIJgwYQLd3d3tzoaZWUeR1PBOcFcNmZlVXKmBQNJUSUskLZV0bJ3pL5Z0laQ/KnWh20ynZmZmNoRKCwRKfbSfRurXZTIwQ9Lkmtk+B8yJiF1Jfbv7UkEzsxYr84xgd2BpRCzLXSScS+q2oCh4vv+VzVm7TrXMzGwdlBkIxrJ63yo9rN6dMqT+zN+j9LSlecDH6q1I0kylp0d19/b2lpFXM7PKandj8QzgjIgYR3pIyFm5O97VRMTsiJgSEVO6uupe/WRmZmupzECwgtX7Lh/Hmv3qH07q2paI+D3pCU1jMDOzlikzEMwHJkmaKGk0qTF4bs08fwX2hed6RdyI3H+4mZm1RmmBIPduOIvUL/ti0tVBCyWdIGlanu1TwBGSbiL1XX5YuPMjM7OWKvXO4oiYR2oELqYdXxheBOxdZh7M1pZqHmroQxQbqTquiwmz4aY2YICDhnWWdl81ZGZmbeZAYGZWca4aMhsEtxvYSOQzAjOzinMgMDOrOAcCM7OKcyAwM6s4BwIzs4pzIDAzqzgHAjOzivN9BNYWvh7fbPjwGYGZWcU5EJiZVZyrhqxyXC1ltjoHAhs2Rtof9Egrj41cDgRm+E/bqs1tBGZmFVdqIJA0VdISSUslHVtn+jck3Zhft0p6qMz8WOeRVn+Z2dArrWpI0ijgNGB/oAeYL2lufk4xABHxicL8HwN2LSs/ZmZWX5lnBLsDSyNiWUQ8BZwLHNjP/DOAc0rMj5mZ1VFmIBgLLC+M9+S0NUjaHpgIXNlg+kxJ3ZK6e3t7hzyjZmZVNlwai6cDF0TEM/UmRsTsiJgSEVO6urpanDUzs5GtzECwAhhfGB+X0+qZjquFbB24Udls7ZUZCOYDkyRNlDSa9Gc/t3YmSTsCWwK/LzEvZmbWQGmBICJWAbOAy4DFwJyIWCjpBEnTCrNOB86N8C081hwf/ZsNrVLvLI6IecC8mrTja8Y/X2YezMysf+5ios3ctYGZtZsDgY1oDrRmAxsul4+amVmbVOqMwEeHZmZr8hmBmVnFORCYmVWcA4GZWcVVqo2gqurddOX2ETPr4zMCM7OKcyAwM6s4Vw1ZqVwtZTb8+YzAzKzifEYwCL4hzUYa79MGDgQdzT9ia8T7hg2Gq4bMzCrOZwS2VnzEaTZy+IzAzKziSg0EkqZKWiJpqaRjG8zzDkmLJC2UdHaZ+bHV+ZGPna/2O/T3aGujtKohSaOA04D9gR5gvqS5EbGoMM8k4Dhg74hYKelFZeWnLK4ieZ4/C7POVOYZwe7A0ohYFhFPAecCB9bMcwRwWkSsBIiI+0rMj5kNMZ+NjAxlBoKxwPLCeE9OK3o58HJJ10q6TtLUeiuSNFNSt6Tu3t7ekrJr4B+2WRW1u7F4fWASsA8wA/i+pC1qZ4qI2RExJSKmdHV1tTiLncV/5FYG71dDZzh+lmUGghXA+ML4uJxW1APMjYinI+IO4FZSYGiZTmls64Q8mllnKjMQzAcmSZooaTQwHZhbM8+FpLMBJI0hVRUtKzFPZmZWo7RAEBGrgFnAZcBiYE5ELJR0gqRpebbLgAckLQKuAj4TEQ+UlSczM1tTqXcWR8Q8YF5N2vGF4QA+mV9mViJf3muNtLux2MzM2sx9DZl1CB/RW1kcCIYh/+DNrJUcCMyGoU4+GOjkvFeV2wjMzCrOZwQ2IB/hVYu/7+pxIGjAPwYzqwoHArMW8gGGDUduIzAzqzifEZhZ6XwmNLw5EJi1mf8kO89I+85cNWRmVnEOBGZmFTdgIJC0jaTTJV2SxydLOrz8rHUuP0TGzDpJM2cEZ5CeG7BdHr8VOLqsDJmZWWs1EwjGRMQc4Fl47oEzz5SaKzMza5lmAsHjkrYGAkDSnsDDpebKzMxappnLRz9JetbwSyVdC3QBh5SaKzMza5kBzwgi4gbg9cBewIeAV0TEzc2sXNJUSUskLZV0bJ3ph0nqlXRjfn1wsAUwMytL7YUfI/Xij2auGvoosGlELIyIW4BNJX2kieVGAacBBwCTgRmSJteZ9byI2CW/fjDI/JuZ2Tpqpo3giIh4qG8kIlYCRzSx3O7A0ohYFhFPAecCB65dNs3MVrcuR+pVOMofjGYCwSjp+Y8qH+mPbmK5scDywnhPTqt1sKSbJV0gaXy9FUmaKalbUndvb28TmzazTrSuf9D+g187zQSCS4HzJO0raV/gnJw2FC4CJkTEq4ArgDPrzRQRsyNiSkRM6erqGqJNm5kZNHfV0L+TGok/nMevAJqpy18BFI/wx+W050TEA4XRHwAnNbHejjTSOqkys5FjwEAQEc8C38mvwZgPTJI0kRQApgPvKs4gaduIuDuPTgMWD3IbZma2jgYMBJL2Bj4PbJ/nFxAR8ZL+louIVZJmkbqnGAX8MCIWSjoB6I6IucDHJU0DVgEPAoetQ1nMzNbgs/GBNVM1dDrwCWABg+xaIiLmAfNq0o4vDB8HHDeYdZqZ2dBqJhA8HBGXlJ4TM7M66l3946P6odVMILhK0snAz4An+xLzHcdmZmvFVTbDRzOBYI/8PqWQFsAbhz47ZmYjR6cEu2auGnpDKzJiZmbt0dTD6yW9FXgFsFFfWkScUFamzMxarVOO3svQTKdz3wXeCXyMdOnooaRLSc3MbARopouJvSLifcDKiPgC8Frg5eVmy8zMWqWZQPC3/P6EpO2Ap4Fty8uSmZm1UjNtBBdL2gI4GbiBdMWQnxtgZlayVrVbNBMIToqIJ4GfSrqY1GD893KyY2ZmrdZM1dDv+wYi4smIeLiYZmZmna3hGYGkfyA9SOYFknYlXTEE8EJg4xbkzczMWqC/qqE3k3oDHQd8necDwaPAf5SbLTOzztLJ9yE0DAQRcSZwpqSDI+KnLcyTmZm1UDNtBOMkvVDJDyTdIOlNpefMzMxaoplA8G8R8QjwJmBr4L3AV0rNlZmZtUwzgaCv5ustwP9GxMJCmpmZdbhmAsECSZeTAsFlkjYDnm1m5ZKmSloiaamkY/uZ72BJIWlKo3nMzKwczdxQdjiwC7AsIp6QtDXwgYEWkjQKOA3YH+gB5kuaGxGLaubbDDgKuH6wmTczs3XX8IxA0o55cJf8/hJJr+b5h9gPZHdgaUQsi4ingHOBA+vMdyLwVXy3splZW/T3h/5JYCbpHoJazTyhbCywvDDew/NPOwMgB5bxEfFLSZ9ptCJJM3NeePGLXzzAZs3MbDD6u49gZn4v5QllktYDTiHdtNaviJgNzAaYMmVKB92mYWY2/PVbxZPbA94F9FUTLQbOjogHm1j3CmB8YXxcTuuzGfBK4GqlW/L+AZgraVpEdDeXfTMzW1f9tRHsBNwC7AbcCtwGvAa4pdB+0J/5wCRJEyWNBqYDc/smRsTDETEmIiZExATgOsBBwMysxfo7IzgROCoi5hQTJR0MfAk4uL8VR8QqSbOAy4BRwA8jYqGkE4DuiJjb3/JmZtYa/QWCf4yIQ2oTI+Knkr7czMojYh4wrybt+Abz7tPMOs3MbGj1d0PZ42s5zczMOkh/ZwQvkvTJOukCukrKj5mZtVh/geD7pCt76vEzi83MRoj+7iP4QiszYmZm7dFMp3NmZjaCORCYmVXcgIFA0sRm0szMrDM1c0ZQ73nFFwx1RszMrD0aNhbnbiReAWwu6aDCpBcCG5WdMTMza43+Lh/dAXgbsAXw9kL6o8ARZWbKzMxap7/LR38B/ELSayPi9y3Mk5mZtVB/VUPHRMRJwLskzaidHhEfLzVnZmbWEv1VDS3O7+4W2sxsBOuvauii/H5m67JjZmat1l/V0EWkZxPXFRHTSsmRmZm1VH9VQ1/L7weRHiP54zw+A7i3zEyZmVnr9Fc19BsASV+PiCmFSRdJcruBmdkI0cydxZtIeknfSO5eYpNmVi5pqqQlkpZKOrbO9CMl/UnSjZKukTS5+aybmdlQ6K9qqM8ngKslLSM9lGZ74EMDLSRpFHAasD/QA8yXNDciFhVmOzsivpvnnwacAkwdXBHMzGxdDBgIIuJSSZOAHXPSnyPiySbWvTuwNCKWAUg6FzgQeC4QRMQjhfk3oZ/GaTMzK0fDqiFJxxRGp0XETfn1ZJMPrx8LLC+M9+S02u18VNLtwEmAb1IzM2ux/toIpheGj6uZNmTVNxFxWkS8FPh34HP15pE0U1K3pO7e3t6h2rSZmdF/IFCD4Xrj9awAxhfGx+W0Rs4F/qXehIiYHRFTImJKV1dXE5s2M7Nm9RcIosFwvfF65gOTJE2UNJp0hjG3OENue+jzVuC2JtZrZmZDqL/G4p0lPUI6+n9BHiaPD/g8gohYJWkWcBkwCvhhRCyUdALQHRFzgVmS9gOeBlYC71+HspiZ2Vro74ayUeu68oiYB8yrSTu+MHzUum7DzMzWjR9eb2ZWcQ4EZmYV50BgZlZxDgRmZhXnQGBmVnEOBGZmFedAYGZWcQ4EZmYV50BgZlZxDgRmZhXnQGBmVnEOBGZmFedAYGZWcQ4EZmYV50BgZlZxDgRmZhXnQGBmVnEOBGZmFVdqIJA0VdISSUslHVtn+iclLZJ0s6RfS9q+zPyYmdmaSgsEkkYBpwEHAJOBGZIm18z2R2BKRLwKuAA4qaz8mJlZfWWeEewOLI2IZRHxFHAucGBxhoi4KiKeyKPXAeNKzI+ZmdVRZiAYCywvjPfktEYOBy6pN0HSTEndkrp7e3uHMItmZjYsGoslvQeYApxcb3pEzI6IKRExpaurq7WZMzMb4dYvcd0rgPGF8XE5bTWS9gM+C7w+Ip4sMT9mZlZHmWcE84FJkiZKGg1MB+YWZ5C0K/A9YFpE3FdiXszMrIHSAkFErAJmAZcBi4E5EbFQ0gmSpuXZTgY2Bc6XdKOkuQ1WZ2ZmJSmzaoiImAfMq0k7vjC8X5nbNzOzgQ2LxmIzM2sfBwIzs4pzIDAzqzgHAjOzinMgMDOrOAcCM7OKcyAwM6s4BwIzs4pzIDAzqzgHAjOzinMgMDOrOAcCM7OKcyAwM6s4BwIzs4pzIDAzqzgHAjOzinMgMDOruFIDgaSpkpZIWirp2DrTXyfpBkmrJB1SZl7MzKy+0gKBpFHAacABwGRghqTJNbP9FTgMOLusfJiZWf/KfGbx7sDSiFgGIOlc4EBgUd8MEXFnnvZsifkwM7N+lFk1NBZYXhjvyWmDJmmmpG5J3b29vUOSOTMzSzqisTgiZkfElIiY0tXV1e7smJmNKGUGghXA+ML4uJxmZmbDSJmBYD4wSdJESaOB6cDcErdnZmZrobRAEBGrgFnAZcBiYE5ELJR0gqRpAJJeI6kHOBT4nqSFZeXHzMzqK/OqISJiHjCvJu34wvB8UpWRmZm1SUc0FpuZWXkcCMzMKs6BwMys4hwIzMwqzoHAzKziHAjMzCrOgcDMrOIcCMzMKs6BwMys4hwIzMwqzoHAzKziHAjMzCrOgcDMrOIcCMzMKs6BwMys4hwIzMwqzoHAzKziHAjMzCqu1EAgaaqkJZKWSjq2zvQNJZ2Xp18vaUKZ+TEzszWVFggkjQJOAw4AJgMzJE2ume1wYGVEvAz4BvDVsvJjZmb1lXlGsDuwNCKWRcRTwLnAgTXzHAicmYcvAPaVpBLzZGZmNdYvcd1jgeWF8R5gj0bzRMQqSQ8DWwP3F2eSNBOYmUcfk7QEGFM732A1Cjn10luQNga4vxXbLmOdNWkNy9KCbTdMW4flB/XdrMN21jmtyXlL2dfWdfl12M6I++3U33LjPDVp+0YTygwEQyYiZgOzi2mSuiNiSpuyNORGUnlGUlnA5RnuRlJ52lWWMquGVgDjC+PjclrdeSStD2wOPFBinszMrEaZgWA+MEnSREmjgenA3Jp55gLvz8OHAFdGRJSYJzMzq1Fa1VCu858FXAaMAn4YEQslnQB0R8Rc4HTgLElLgQdJwaJZsweepaOMpPKMpLKAyzPcjaTytKUs8gG4mVm1+c5iM7OKcyAwM6u4jggEkn4o6T5JtxTStpJ0haTb8vuW7cxjsySNl3SVpEWSFko6Kqd3ank2kvQHSTfl8nwhp0/M3YYszd2IjG53XpslaZSkP0q6OI93clnulPQnSTdK6s5pHbmvAUjaQtIFkv4sabGk13ZqeSTtkL+Xvtcjko5uR3k6IhAAZwBTa9KOBX4dEZOAX+fxTrAK+FRETAb2BD6au97o1PI8CbwxInYGdgGmStqT1F3IN3L3IStJ3Yl0iqOAxYXxTi4LwBsiYpfC9emduq8BfBO4NCJ2BHYmfU8dWZ6IWJK/l12A3YAngJ/TjvJEREe8gAnALYXxJcC2eXhbYEm787iW5foFsP9IKA+wMXAD6Q7y+4H1c/prgcvanb8myzAu//jeCFwMqFPLkvN7JzCmJq0j9zXSfUZ3kC9y6fTy1JThTcC17SpPp5wR1LNNRNydh+8BtmlnZtZG7m11V+B6Org8uSrlRuA+4ArgduChiFiVZ+khdSfSCf4bOAZ4No9vTeeWBSCAyyUtyF21QOfuaxOBXuBHueruB5I2oXPLUzQdOCcPt7w8nRwInhMpdHbUdbCSNgV+ChwdEY8Up3VaeSLimUint+NInQ3u2OYsrRVJbwPui4gF7c7LEPqniHg1qRfgj0p6XXFih+1r6wOvBr4TEbsCj1NTbdJh5QEgtzlNA86vndaq8nRyILhX0rYA+f2+NuenaZI2IAWBn0TEz3Jyx5anT0Q8BFxFqj7ZIncbAvW7FxmO9gamSbqT1FvuG0l10p1YFgAiYkV+v49U/7w7nbuv9QA9EXF9Hr+AFBg6tTx9DgBuiIh783jLy9PJgaDYPcX7SXXtw17uZvt0YHFEnFKY1Knl6ZK0RR5+Aam9YzEpIBySZ+uI8kTEcRExLiImkE7Vr4yId9OBZQGQtImkzfqGSfXQt9Ch+1pE3AMsl7RDTtoXWESHlqdgBs9XC0E7ytPuRpImG1LOAe4GniYdFRxOqrv9NXAb8Ctgq3bns8my/BPpVO9m4Mb8eksHl+dVwB9zeW4Bjs/pLwH+ACwlnfJu2O68DrJc+wAXd3JZcr5vyq+FwGdzekfuaznvuwDdeX+7ENiyw8uzCamjzc0LaS0vj7uYMDOruE6uGjIzsyHgQGBmVnEOBGZmFedAYGZWcQ4EZmYV50BglSfpsZLXf7SkjVu1PbPBciAwK9/RpA75zIYlBwKzOiS9VNKlubO230raMaefIelUSb+TtEzSITl9PUnfzv3kXyFpnqRDJH0c2A64StJVhfV/KT/D4TpJ2+S0QyXdktP/rx3ltmpyIDCrbzbwsYjYDfg08O3CtG1Jd4i/DfhKTjuI1FX6ZOC9pP6WiIhTgbtIzwR4Q553E+C6SM9w+D/giJx+PPDmnD6tnGKZrWn9gWcxq5bcM+xewPmpaygANizMcmFEPAss6juaJwWG83P6PcWj/zqeIj3rAGABqX8mgGuBMyTNAX5Wb0GzMjgQmK1pPdIzCHZpMP3JwrAazNOfp+P5vl2eIf8OI+JISXsAbwUWSNotIh5Yi/WbDYqrhsxqRHo+xB2SDoXUY6yknQdY7Frg4NxWsA2p07o+jwKbDbRdSS+NiOsj4njSA1jGr8sJxL8AAACSSURBVFUBzAbJZwRmsLGknsL4KcC7ge9I+hywAen5BDf1s46f8ny3yMtJj+x8OE+bDVwq6a5CO0E9J0uaRDrL+PUA2zMbMu591GyISNo0Ih6TtDWp2+q9I/Whbzas+YzAbOhcnB/SMxo40UHAOoXPCMzMKs6NxWZmFedAYGZWcQ4EZmYV50BgZlZxDgRmZhX3/wGlqoxM8vroagAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["## Sequence Matching"],"metadata":{"id":"ntfio0MIMztS"}},{"cell_type":"code","source":["def match_sequences(original_phoneme_sequence, original_words, generated_phoneme_sequence):\n","  generated_idxs = {token : idx for idx, token in enumerate(generated)}\n","  original_matches = []\n","  for token in original:\n","    if token in generated_idxs: \n","      original_matches.append(generated_idxs[token])\n","    else:\n","      original_matches.append(-1)\n","  return original_matches"],"metadata":{"id":"FqCzzh_cM1xd"},"execution_count":null,"outputs":[]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":["ySPgc0-17d8Y","f6KdcHls9kur","ntfio0MIMztS"],"name":"text2phoneme_transformer.ipynb","provenance":[{"file_id":"1HFbc73QbCFwt7iaNJHX5fQdeLNaVtuY1","timestamp":1639546183567}]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"bc5d01f9505d4c5692b116bf5cbfd921":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_c28803a8b7054413860d06836194034c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_41b8c2ac6da44b7399b1076305509855","IPY_MODEL_61865673211f4c6ea73ff6f346e171a1","IPY_MODEL_b34d81cfe056497997680a584df615ac"]}},"c28803a8b7054413860d06836194034c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"41b8c2ac6da44b7399b1076305509855":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_4b283511714842559d41fb8f31f01bb2","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"Downloading: 100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_02904e539c3a494393986af5cee371b9"}},"61865673211f4c6ea73ff6f346e171a1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_73482b9190a54436890d91e304185d81","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":869007403,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":869007403,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6f23ddc39ea2440ca00a3c646aee962d"}},"b34d81cfe056497997680a584df615ac":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_272024c6238a4b1cbe888d547627898f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 869M/869M [00:45&lt;00:00, 20.0MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0f5005881e4c46f48c963ba266144fbc"}},"4b283511714842559d41fb8f31f01bb2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"02904e539c3a494393986af5cee371b9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"73482b9190a54436890d91e304185d81":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6f23ddc39ea2440ca00a3c646aee962d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"272024c6238a4b1cbe888d547627898f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0f5005881e4c46f48c963ba266144fbc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d06641948886478d8bf7185c22f2591b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_14894674f58c417bb3c3a5d12205cffe","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_edeb40b5644741fe9b9249cd1321907d","IPY_MODEL_6e4b968aeb5a4a849d37889eb04fb3ba","IPY_MODEL_0b82c21e2f9843b7bc7ec7396d901a79"]}},"14894674f58c417bb3c3a5d12205cffe":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"edeb40b5644741fe9b9249cd1321907d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c0b7d520a7ae476890a104e36bb7fef0","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a47bec8da73943208bc2b4dda0abc0d5"}},"6e4b968aeb5a4a849d37889eb04fb3ba":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_7bcf2aae88e342dcb0a81d122f831521","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"info","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c9fb8e4168bd4a8ebc1b2e4e685034d5"}},"0b82c21e2f9843b7bc7ec7396d901a79":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3eb9ddcd25774487b656cdb048f25720","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 4603/0 [00:04&lt;00:00, 1233.36 examples/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_392598e80294491ba1a1eed8de5b3580"}},"c0b7d520a7ae476890a104e36bb7fef0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a47bec8da73943208bc2b4dda0abc0d5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7bcf2aae88e342dcb0a81d122f831521":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c9fb8e4168bd4a8ebc1b2e4e685034d5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":"20px","min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3eb9ddcd25774487b656cdb048f25720":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"392598e80294491ba1a1eed8de5b3580":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3c1a14e1b4734d7db2b9a1471610d50f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9cf4d0271d2f4eeb8343a07d12f025b6","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_31a3b4ccbd554fca8ca880dde2ef37cb","IPY_MODEL_b9f23d9094ba42458f84e7d35d178d60","IPY_MODEL_fef496f780f44745ad6a4bfd5b6718b7"]}},"9cf4d0271d2f4eeb8343a07d12f025b6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"31a3b4ccbd554fca8ca880dde2ef37cb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ed578ff7c89e4839a4b3548764309757","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_81bfe3fef60942e4bdacfda857bad23b"}},"b9f23d9094ba42458f84e7d35d178d60":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_506b9c939a60448881c585bd54a3af14","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"info","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8dacfff7491c4b11a9c31940b8d66310"}},"fef496f780f44745ad6a4bfd5b6718b7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_fd48d8b2ea884309bf02e8fdf40922a7","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1633/0 [00:01&lt;00:00, 1230.21 examples/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c8d5396762d347b6b3f0f394263ca249"}},"ed578ff7c89e4839a4b3548764309757":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"81bfe3fef60942e4bdacfda857bad23b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"506b9c939a60448881c585bd54a3af14":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"8dacfff7491c4b11a9c31940b8d66310":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":"20px","min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"fd48d8b2ea884309bf02e8fdf40922a7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c8d5396762d347b6b3f0f394263ca249":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"76b19283c866471a96fdfb74104f8c95":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_79c5f718ebc048b9bd14e50e15556d6a","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_f220dbc1f7cf460494fa380ea9187c03","IPY_MODEL_ef8e8158fcc24f8797b126543d51765f","IPY_MODEL_af57205ab6c54e9eac09dd515eff85f2"]}},"79c5f718ebc048b9bd14e50e15556d6a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f220dbc1f7cf460494fa380ea9187c03":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_78dd47cd43864c228f36d05ebe845a3c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_85d7817bcf554456868e1a6e762e627f"}},"ef8e8158fcc24f8797b126543d51765f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_fc48e1f1b2534f94838c4f56c98fa806","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":2,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_426e2d7bb3bd4bd4beb8cbed3ec22320"}},"af57205ab6c54e9eac09dd515eff85f2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3fbd2ec4ae8b4373bca3ebac00230727","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2/2 [00:00&lt;00:00, 60.85it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_09ee26670cff4b908ea97fbec065c442"}},"78dd47cd43864c228f36d05ebe845a3c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"85d7817bcf554456868e1a6e762e627f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"fc48e1f1b2534f94838c4f56c98fa806":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"426e2d7bb3bd4bd4beb8cbed3ec22320":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3fbd2ec4ae8b4373bca3ebac00230727":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"09ee26670cff4b908ea97fbec065c442":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"nbformat":4,"nbformat_minor":0}